From 1c0aca25a3e1c816794918b629db8b6bddfda1f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20de=20la=20Pen=CC=83a?= Date: Fri, 4 Sep 2026 09:19:18 +0100 Subject: [PATCH 1/7] refactor: Migrate legacy form validation to `buildValidator()` Replaces the deprecated `set_rules`/`set_message`/`run()` calls in admin account creation, social sign-on data capture, MFA device setup and MFA question setup with `buildValidator()`. Where the old code relied on `trim` being written back into `$_POST`, the success paths now read `getValidatedData()`. Forced `fv_*` message overrides are dropped where they matched the validator's defaults. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 5d25d8a181c570d02e087aeca2282a2f63a33bb5) --- auth/controllers/Login.php | 66 +++++++++++++++++-------------- auth/controllers/MfaDevice.php | 30 ++++++++------ auth/controllers/MfaQuestion.php | 60 ++++++++++++---------------- src/Admin/Controller/Accounts.php | 47 ++++++++++++---------- 4 files changed, 107 insertions(+), 96 deletions(-) diff --git a/auth/controllers/Login.php b/auth/controllers/Login.php index 371db08c..d91f7de6 100644 --- a/auth/controllers/Login.php +++ b/auth/controllers/Login.php @@ -948,63 +948,69 @@ protected function socialSignOnRequestData(array &$aRequiredData): void /** @var FormValidation $oFormValidation */ $oFormValidation = Factory::service('FormValidation'); + $aRules = []; + if (isset($aRequiredData['email'])) { - $oFormValidation->set_rules('email', 'email', 'trim|required|valid_email|is_unique[' . \Nails\Config::get('NAILS_DB_PREFIX') . 'user_email.email]'); + $aRules['email'] = [ + 'trim', + FormValidation::RULE_REQUIRED, + FormValidation::RULE_VALID_EMAIL, + FormValidation::rule(FormValidation::RULE_IS_UNIQUE, \Nails\Config::get('NAILS_DB_PREFIX') . 'user_email', 'email'), + ]; } if (isset($aRequiredData['username'])) { - $oFormValidation->set_rules('username', 'username', 'trim|required|is_unique[' . \Nails\Config::get('NAILS_DB_PREFIX') . 'user.username]'); + $aRules['username'] = [ + 'trim', + FormValidation::RULE_REQUIRED, + FormValidation::rule(FormValidation::RULE_IS_UNIQUE, \Nails\Config::get('NAILS_DB_PREFIX') . 'user', 'username'), + ]; } if (empty($aRequiredData['first_name'])) { - $oFormValidation->set_rules('first_name', '', 'trim|required'); + $aRules['first_name'] = ['trim', FormValidation::RULE_REQUIRED]; } if (empty($aRequiredData['last_name'])) { - $oFormValidation->set_rules('last_name', '', 'trim|required'); + $aRules['last_name'] = ['trim', FormValidation::RULE_REQUIRED]; } - $oFormValidation->set_message('required', lang('fv_required')); - $oFormValidation->set_message('valid_email', lang('fv_valid_email')); + $sIsUniqueMessage = match (\Nails\Config::get('APP_NATIVE_LOGIN_USING')) { + 'EMAIL' => lang('fv_email_already_registered', siteUrl('auth/password/forgotten')), + 'USERNAME' => lang('fv_username_already_registered', siteUrl('auth/password/forgotten')), + default => lang('fv_identity_already_registered', siteUrl('auth/password/forgotten')), + }; - if (\Nails\Config::get('APP_NATIVE_LOGIN_USING') == 'EMAIL') { - $oFormValidation->set_message( - 'is_unique', - lang('fv_email_already_registered', siteUrl('auth/password/forgotten')) - ); - } elseif (\Nails\Config::get('APP_NATIVE_LOGIN_USING') == 'USERNAME') { - $oFormValidation->set_message( - 'is_unique', - lang('fv_username_already_registered', siteUrl('auth/password/forgotten')) - ); - } else { - $oFormValidation->set_message( - 'is_unique', - lang('fv_identity_already_registered', siteUrl('auth/password/forgotten')) - ); - } + try { + + $oValidator = $oFormValidation + ->buildValidator($aRules, [ + FormValidation::RULE_IS_UNIQUE => $sIsUniqueMessage, + ]) + ->setLabels(['email' => 'email', 'username' => 'username']) + ->run(); - if ($oFormValidation->run()) { + // Valid! Ensure required data is set correctly then allow system to move on. + $aPost = $oValidator->getValidatedData(); - // Valid!Ensure required data is set correctly then allow system to move on. if (isset($aRequiredData['email'])) { - $aRequiredData['email'] = $oInput->post('email'); + $aRequiredData['email'] = $aPost['email'] ?? null; } if (isset($aRequiredData['username'])) { - $aRequiredData['username'] = $oInput->post('username'); + $aRequiredData['username'] = $aPost['username'] ?? null; } if (empty($aRequiredData['first_name'])) { - $aRequiredData['first_name'] = $oInput->post('first_name'); + $aRequiredData['first_name'] = $aPost['first_name'] ?? null; } if (empty($aRequiredData['last_name'])) { - $aRequiredData['last_name'] = $oInput->post('last_name'); + $aRequiredData['last_name'] = $aPost['last_name'] ?? null; } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); $this->socialSignOnRequestDataForm($aRequiredData, $provider); } diff --git a/auth/controllers/MfaDevice.php b/auth/controllers/MfaDevice.php index e893123b..68fb126d 100644 --- a/auth/controllers/MfaDevice.php +++ b/auth/controllers/MfaDevice.php @@ -11,6 +11,7 @@ */ use Nails\Common\Exception\FactoryException; +use Nails\Common\Exception\ValidationException; use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; use Nails\Factory; @@ -82,12 +83,14 @@ protected function setupDevice() /** @var FormValidation $oFormValidation */ $oFormValidation = Factory::service('FormValidation'); - $oFormValidation->set_rules('mfa_secret', '', 'required'); - $oFormValidation->set_rules('mfa_code', '', 'required'); + try { - $oFormValidation->set_message('required', lang('fv_required')); - - if ($oFormValidation->run()) { + $oFormValidation + ->buildValidator([ + 'mfa_secret' => [FormValidation::RULE_REQUIRED], + 'mfa_code' => [FormValidation::RULE_REQUIRED], + ]) + ->run(); $sSecret = $oInput->post('mfa_secret'); $sMfaCode = $oInput->post('mfa_code'); @@ -108,8 +111,8 @@ protected function setupDevice() $this->oUserFeedback->error('Sorry, that code failed to validate. Please try again.'); } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); } } @@ -152,10 +155,13 @@ protected function requestCode() /** @var FormValidation $oFormValidation */ $oFormValidation = Factory::service('FormValidation'); - $oFormValidation->set_rules('mfa_code', '', 'required'); - $oFormValidation->set_message('required', lang('fv_required')); + try { - if ($oFormValidation->run()) { + $oFormValidation + ->buildValidator([ + 'mfa_code' => [FormValidation::RULE_REQUIRED], + ]) + ->run(); /** @var Authentication $oAuthService */ $oAuthService = Factory::service('Authentication', Constants::MODULE_SLUG); @@ -171,8 +177,8 @@ protected function requestCode() )); } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); } } diff --git a/auth/controllers/MfaQuestion.php b/auth/controllers/MfaQuestion.php index 57255f67..d827e66d 100644 --- a/auth/controllers/MfaQuestion.php +++ b/auth/controllers/MfaQuestion.php @@ -15,6 +15,7 @@ use Nails\Auth\Service\Authentication; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\NailsException; +use Nails\Common\Exception\ValidationException; use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; use Nails\Factory; @@ -112,44 +113,35 @@ public function index() /** @var FormValidation $oFormValidation */ $oFormValidation = Factory::service('FormValidation'); - for ($i = 0; $i < $this->data['num_questions']; $i++) { - - $oFormValidation->set_rules( - 'question[' . $i . '][question]', - '', - 'required|is_natural_no_zero' - ); + $aRules = []; - $oFormValidation->set_rules( - 'question[' . $i . '][answer]', - '', - 'trim|required' - ); + for ($i = 0; $i < $this->data['num_questions']; $i++) { + $aRules['question[' . $i . '][question]'] = [ + FormValidation::RULE_REQUIRED, + FormValidation::RULE_IS_NATURAL_NO_ZERO, + ]; + $aRules['question[' . $i . '][answer]'] = ['trim', FormValidation::RULE_REQUIRED]; } for ($i = 0; $i < $this->data['num_custom_questions']; $i++) { + $aRules['custom_question[' . $i . '][question]'] = ['trim', FormValidation::RULE_REQUIRED]; + $aRules['custom_question[' . $i . '][answer]'] = ['trim', FormValidation::RULE_REQUIRED]; + } - $oFormValidation->set_rules( - 'custom_question[' . $i . '][question]', - '', - 'trim|required' - ); + try { - $oFormValidation->set_rules( - 'custom_question[' . $i . '][answer]', - '', - 'trim|required' + $oValidator = $oFormValidation->buildValidator( + $aRules, + [FormValidation::RULE_IS_NATURAL_NO_ZERO => lang('fv_required')] ); - } - - $oFormValidation->set_message('required', lang('fv_required')); - $oFormValidation->set_message('is_natural_no_zero', lang('fv_required')); + $oValidator->run(); - if ($oFormValidation->run()) { + // The validated data carries the trimmed values + $aPost = $oValidator->getValidatedData(); // Make sure that we have different questions $aQuestionIndex = []; - $aQuestion = array_filter((array) $oInput->post('question', true)); + $aQuestion = array_filter((array) ($aPost['question'] ?? [])); $bError = false; foreach ($aQuestion as $q) { @@ -163,7 +155,7 @@ public function index() } $aQuestionIndex = []; - $aQuestion = array_filter((array) $oInput->post('custom_question', true)); + $aQuestion = array_filter((array) ($aPost['custom_question'] ?? [])); foreach ($aQuestion as $q) { if (array_search($q['question'], $aQuestionIndex) === false) { @@ -179,9 +171,9 @@ public function index() // Good arrows. Save questions $aData = []; - if ($oInput->post('question', true)) { + if (!empty($aPost['question'])) { - foreach ($oInput->post('question', true) as $q) { + foreach ($aPost['question'] as $q) { $oTemp = new stdClass(); @@ -196,8 +188,8 @@ public function index() } } - if ($oInput->post('custom_question', true)) { - foreach ((array) $oInput->post('custom_question', true) as $aQuestion) { + if (!empty($aPost['custom_question'])) { + foreach ((array) $aPost['custom_question'] as $aQuestion) { $aData[] = (object) [ 'question' => trim($aQuestion['question']), 'answer' => $aQuestion['answer'], @@ -228,8 +220,8 @@ public function index() $this->oUserFeedback->error(lang('auth_twofactor_question_unique')); } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); } } diff --git a/src/Admin/Controller/Accounts.php b/src/Admin/Controller/Accounts.php index bbca9ce8..0f07710e 100755 --- a/src/Admin/Controller/Accounts.php +++ b/src/Admin/Controller/Accounts.php @@ -366,29 +366,36 @@ public function create(): void /** @var FormValidation $oFormValidation */ $oFormValidation = Factory::service('FormValidation'); - // Set rules - $oFormValidation->set_rules('group_id', '', 'required|is_natural_no_zero'); - $oFormValidation->set_rules('password', '', ''); - $oFormValidation->set_rules('send_activation', '', ''); - $oFormValidation->set_rules('temp_pw', '', ''); - $oFormValidation->set_rules('first_name', '', 'required|max_length[150]'); - $oFormValidation->set_rules('last_name', '', 'required|max_length[150]'); - $oFormValidation->set_rules('email', '', 'required|valid_email|is_unique[' . Config::get('NAILS_DB_PREFIX') . 'user_email.email]|max_length[255]'); + $aRules = [ + 'group_id' => [FormValidation::RULE_REQUIRED, FormValidation::RULE_IS_NATURAL_NO_ZERO], + 'first_name' => [FormValidation::RULE_REQUIRED, FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150)], + 'last_name' => [FormValidation::RULE_REQUIRED, FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150)], + 'email' => [ + FormValidation::RULE_REQUIRED, + FormValidation::RULE_VALID_EMAIL, + FormValidation::rule(FormValidation::RULE_IS_UNIQUE, Config::get('NAILS_DB_PREFIX') . 'user_email', 'email'), + FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 255), + ], + ]; if (in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['BOTH', 'USERNAME'])) { - $oFormValidation->set_rules('username', '', 'required|max_length[150]|alpha_dash_period|is_unique[' . Config::get('NAILS_DB_PREFIX') . 'user.username]'); + $aRules['username'] = [ + FormValidation::RULE_REQUIRED, + FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150), + FormValidation::RULE_ALPHA_DASH_PERIOD, + FormValidation::rule(FormValidation::RULE_IS_UNIQUE, Config::get('NAILS_DB_PREFIX') . 'user', 'username'), + ]; } - // Set messages - $oFormValidation->set_message('required', lang('fv_required')); - $oFormValidation->set_message('min_length', lang('fv_min_length')); - $oFormValidation->set_message('alpha_dash_period', lang('fv_alpha_dash_period')); - $oFormValidation->set_message('is_natural_no_zero', lang('fv_required')); - $oFormValidation->set_message('valid_email', lang('fv_valid_email')); - $oFormValidation->set_message('is_unique', lang('fv_email_already_registered')); + try { + + $oFormValidation + ->buildValidator($aRules, [ + FormValidation::RULE_IS_NATURAL_NO_ZERO => lang('fv_required'), + FormValidation::RULE_IS_UNIQUE => lang('fv_email_already_registered'), + ]) + ->run(); - // Execute - if ($oFormValidation->run()) { // Success $aData = [ @@ -463,8 +470,8 @@ public function create(): void )); } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); } } From 1b5c1273cd3ff5ed222a60597e5b409a8e7e3318 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20de=20la=20Pen=CC=83a?= Date: Fri, 4 Sep 2026 09:19:18 +0100 Subject: [PATCH 2/7] refactor: Load language lines via the Translation service Replaces `get_instance()->lang->load(...)` with `Factory::service('Translation')->load(...)`, which works outside a CodeIgniter request. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 63bca4ff7bccc3c4cf7c35278ad941e60cc74a77) --- src/Admin/Controller/Accounts.php | 2 +- src/Controller/Base.php | 2 +- src/Event/Listener/User/Init.php | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Admin/Controller/Accounts.php b/src/Admin/Controller/Accounts.php index 0f07710e..ad38f570 100755 --- a/src/Admin/Controller/Accounts.php +++ b/src/Admin/Controller/Accounts.php @@ -274,7 +274,7 @@ public function __construct() // -------------------------------------------------------------------------- - get_instance()->lang->load('admin_accounts'); + Factory::service('Translation')->load('admin_accounts'); /** @var ChangeLog oChangeLogModel */ $this->oChangeLogModel = Factory::model('ChangeLog', \Nails\Admin\Constants::MODULE_SLUG); } diff --git a/src/Controller/Base.php b/src/Controller/Base.php index f39deef9..b73d809b 100644 --- a/src/Controller/Base.php +++ b/src/Controller/Base.php @@ -28,7 +28,7 @@ public function __construct() parent::__construct(); $oConfig = Factory::service('Config'); $oConfig->load('auth/auth'); - get_instance()->lang->load('auth/auth'); + Factory::service('Translation')->load('auth'); } // -------------------------------------------------------------------------- diff --git a/src/Event/Listener/User/Init.php b/src/Event/Listener/User/Init.php index 99540a71..873f9601 100644 --- a/src/Event/Listener/User/Init.php +++ b/src/Event/Listener/User/Init.php @@ -151,7 +151,7 @@ protected function checkUserIsSuspended(): self /** @var Authentication $oAuthService */ $oAuthService = Factory::service('Authentication', Constants::MODULE_SLUG); - get_instance()->lang->load('auth/auth'); + Factory::service('Translation')->load('auth'); $oAuthService->logout(); From 48222e4f534315b3bcfc0692a57bcd15b9b21a10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20de=20la=20Pen=CC=83a?= Date: Fri, 4 Sep 2026 09:19:18 +0100 Subject: [PATCH 3/7] refactor: Read cross-field values via the validation `Context` in import rules The password rule read `group_id` through the FormValidation service's CodeIgniter `validation_data` property; it now receives a `Context` and reads the row being validated directly, which also removes the last reason the rules needed the service at all. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 3a8dd87ad51f274761ec02e194dcc8cab6f245d0) --- src/Service/User/Import.php | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/Service/User/Import.php b/src/Service/User/Import.php index 1767ea05..52486bad 100644 --- a/src/Service/User/Import.php +++ b/src/Service/User/Import.php @@ -21,6 +21,7 @@ use Nails\Common\Factory\Model\Field; use Nails\Common\Service\DateTime; use Nails\Common\Service\FormValidation; +use Nails\Common\Validation\Context; use Nails\Config; use Nails\Factory; @@ -85,8 +86,6 @@ public function getValidationRules(string $key): array $oUserPasswordModel = Factory::model('UserPassword', Constants::MODULE_SLUG); /** @var DateTime $oDateTimeService */ $oDateTimeService = Factory::service('DateTime'); - /** @var FormValidation $oFormValidationService */ - $oFormValidationService = Factory::service('FormValidation'); return match ($key) { 'email' => array_filter([ @@ -138,13 +137,13 @@ function ($groupId) use ($oUserGroupModel) { }, ], 'password' => [ - function ($password) use ($oUserPasswordModel, $oUserGroupModel, $oFormValidationService) { + function ($password, Context $oContext) use ($oUserPasswordModel, $oUserGroupModel) { if (!$password) { return; } - $groupId = $oFormValidationService->validation_data['group_id'] ?? null; + $groupId = $oContext->getValue('group_id'); $group = $oUserGroupModel->getById((int) $groupId) ?? $oUserGroupModel->getDefaultGroup(); if (!$oUserPasswordModel->isAcceptable($group, $password)) { From fe2cb24132dc48c3eb530e9352efe5de961579f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20de=20la=20Pen=CC=83a?= Date: Fri, 4 Sep 2026 09:45:51 +0100 Subject: [PATCH 4/7] feat: Add `Identity` and `Identifier` validators and use them across the auth forms `Nails\Auth\Validator\User\Identity` owns the email/username rules (required, format, length, uniqueness, per `APP_NATIVE_LOGIN_USING`) and the "already registered" message; `Identifier` owns the login identifier rule. Registration, social sign-on data capture, admin account creation, login and forgotten-password previously each rebuilt these rules with small differences; they now extend the shared validators via `addRules()`/`setRules()`. Both classes are unit tested with the uniqueness rule stubbed, so no database is needed. Behaviour notes: registration and social sign-on now also enforce `alpha_dash_period` and `max_length[150]` on usernames (previously admin-only); admin account creation's "already registered" message now carries the forgotten-password link and uses the `auth_register_*_is_unique` lines rather than `fv_*_already_registered`. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 7a47ee8084961fd6959fb26dce8c0b063740e7ff) --- auth/controllers/Login.php | 56 +++------- auth/controllers/PasswordForgotten.php | 16 +-- auth/controllers/Register.php | 44 ++------ src/Admin/Controller/Accounts.php | 37 ++----- src/Validator/User/Identifier.php | 34 +++++++ src/Validator/User/Identity.php | 129 ++++++++++++++++++++++++ tests/Validator/User/IdentifierTest.php | 37 +++++++ tests/Validator/User/IdentityTest.php | 123 ++++++++++++++++++++++ 8 files changed, 354 insertions(+), 122 deletions(-) create mode 100644 src/Validator/User/Identifier.php create mode 100644 src/Validator/User/Identity.php create mode 100644 tests/Validator/User/IdentifierTest.php create mode 100644 tests/Validator/User/IdentityTest.php diff --git a/auth/controllers/Login.php b/auth/controllers/Login.php index d91f7de6..225e53e2 100644 --- a/auth/controllers/Login.php +++ b/auth/controllers/Login.php @@ -22,6 +22,8 @@ use Nails\Auth\Resource; use Nails\Auth\Service\Authentication; use Nails\Auth\Service\SocialSignOn; +use Nails\Auth\Validator\User\Identifier; +use Nails\Auth\Validator\User\Identity; use Nails\Cdn\Service\Cdn; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\ValidationException; @@ -113,8 +115,6 @@ public function index() $oInput = Factory::service('Input'); /** @var \App\Auth\Model\User $oUserModel */ $oUserModel = Factory::model('User', Constants::MODULE_SLUG); - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); /** @var Authentication $oAuthService */ $oAuthService = Factory::service('Authentication', Constants::MODULE_SLUG); /** @var SocialSignOn $oSocial */ @@ -128,20 +128,9 @@ public function index() try { - $oFormValidation - ->buildValidator([ - 'identifier' => array_values(array_filter([ - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'EMAIL' ? [ - $oFormValidation::RULE_REQUIRED, - $oFormValidation::RULE_VALID_EMAIL, - ] : null, - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'USERNAME' ? [$oFormValidation::RULE_REQUIRED] : null, - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'BOTH' ? [$oFormValidation::RULE_REQUIRED] : null, - ]))[0], - 'password' => [$oFormValidation::RULE_REQUIRED], - 'remember' => [], - ]) - ->run(); + (new Identifier()) + ->addRules(['password' => [FormValidation::RULE_REQUIRED]]) + ->run($oInput->post()); if (appSetting('user_login_captcha_enabled', 'auth')) { if (!$oCaptchaService->verify()) { @@ -945,26 +934,15 @@ protected function socialSignOnRequestData(array &$aRequiredData): void $oInput = Factory::service('Input'); if ($oInput->post()) { - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); - + // Only capture the identity fields we've been asked for $aRules = []; - if (isset($aRequiredData['email'])) { - $aRules['email'] = [ - 'trim', - FormValidation::RULE_REQUIRED, - FormValidation::RULE_VALID_EMAIL, - FormValidation::rule(FormValidation::RULE_IS_UNIQUE, \Nails\Config::get('NAILS_DB_PREFIX') . 'user_email', 'email'), - ]; + if (!isset($aRequiredData['email'])) { + $aRules['email'] = []; } - if (isset($aRequiredData['username'])) { - $aRules['username'] = [ - 'trim', - FormValidation::RULE_REQUIRED, - FormValidation::rule(FormValidation::RULE_IS_UNIQUE, \Nails\Config::get('NAILS_DB_PREFIX') . 'user', 'username'), - ]; + if (!isset($aRequiredData['username'])) { + $aRules['username'] = []; } if (empty($aRequiredData['first_name'])) { @@ -975,20 +953,12 @@ protected function socialSignOnRequestData(array &$aRequiredData): void $aRules['last_name'] = ['trim', FormValidation::RULE_REQUIRED]; } - $sIsUniqueMessage = match (\Nails\Config::get('APP_NATIVE_LOGIN_USING')) { - 'EMAIL' => lang('fv_email_already_registered', siteUrl('auth/password/forgotten')), - 'USERNAME' => lang('fv_username_already_registered', siteUrl('auth/password/forgotten')), - default => lang('fv_identity_already_registered', siteUrl('auth/password/forgotten')), - }; - try { - $oValidator = $oFormValidation - ->buildValidator($aRules, [ - FormValidation::RULE_IS_UNIQUE => $sIsUniqueMessage, - ]) + $oValidator = (new Identity()) + ->setRules($aRules) ->setLabels(['email' => 'email', 'username' => 'username']) - ->run(); + ->run($oInput->post()); // Valid! Ensure required data is set correctly then allow system to move on. $aPost = $oValidator->getValidatedData(); diff --git a/auth/controllers/PasswordForgotten.php b/auth/controllers/PasswordForgotten.php index 58eed802..e41c79a2 100644 --- a/auth/controllers/PasswordForgotten.php +++ b/auth/controllers/PasswordForgotten.php @@ -17,12 +17,12 @@ use Nails\Auth\Model\User; use Nails\Auth\Model\User\Password; use Nails\Auth\Service\Authentication; +use Nails\Auth\Validator\User\Identifier; use Nails\Common\Exception\Encrypt\DecodeException; use Nails\Common\Exception\EnvironmentException; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\NailsException; use Nails\Common\Service\Config; -use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; use Nails\Common\Service\Uri; use Nails\Factory; @@ -53,8 +53,6 @@ public function index() { /** @var Input $oInput */ $oInput = Factory::service('Input'); - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); /** @var Config $oConfig */ $oConfig = Factory::service('Config'); /** @var Password $oUserPasswordModel */ @@ -86,17 +84,7 @@ public function index() // -------------------------------------------------------------------------- - $aRules = array_filter([ - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'EMAIL' ? ['required', 'valid_email'] : null, - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'USERNAME' ? ['required'] : null, - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'BOTH' ? ['required'] : null, - ]); - - $oFormValidation - ->buildValidator([ - 'identifier' => reset($aRules), - ]) - ->run(); + (new Identifier())->run($oInput->post()); if (appSetting('user_password_reset_captcha_enabled', 'auth')) { if (!$oCaptchaService->verify()) { diff --git a/auth/controllers/Register.php b/auth/controllers/Register.php index d5432b8e..8054a061 100644 --- a/auth/controllers/Register.php +++ b/auth/controllers/Register.php @@ -15,6 +15,7 @@ use Nails\Auth\Model\User\Group; use Nails\Auth\Model\User\Password; use Nails\Auth\Service\SocialSignOn; +use Nails\Auth\Validator\User\Identity; use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; use Nails\Common\Service\Session; @@ -57,8 +58,6 @@ public function index() { /** @var Session $oSession */ $oSession = Factory::service('Session'); - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); /** @var Input $oInput */ $oInput = Factory::service('Input'); /** @var \Nails\Auth\Model\User $oUserModel */ @@ -88,42 +87,13 @@ public function index() try { - $oFormValidation - ->buildValidator([ - 'first_name' => [$oFormValidation::RULE_REQUIRED], - 'last_name' => [$oFormValidation::RULE_REQUIRED], - 'password' => [$oFormValidation::RULE_REQUIRED], - 'email' => in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['EMAIL', 'BOTH']) ? [ - $oFormValidation::RULE_REQUIRED, - $oFormValidation::RULE_VALID_EMAIL, - $oFormValidation::rule( - $oFormValidation::RULE_IS_UNIQUE, $oUserEmailModel->getTableName(), 'email' - ), - ] : [], - 'username' => in_array(Config::get('APP_NATIVE_LOGIN_USING'), [ - 'USERNAME', - 'BOTH', - ]) ? [ - $oFormValidation::RULE_REQUIRED, - $oFormValidation::rule( - $oFormValidation::RULE_IS_UNIQUE, $oUserModel->getTableName(), 'username' - ), - ] : [], + (new Identity()) + ->addRules([ + 'first_name' => [FormValidation::RULE_REQUIRED], + 'last_name' => [FormValidation::RULE_REQUIRED], + 'password' => [FormValidation::RULE_REQUIRED], ]) - ->setMessages([ - $oFormValidation::RULE_IS_UNIQUE => implode('', [ - Config::get('APP_NATIVE_LOGIN_USING') === 'EMAIL' - ? lang('auth_register_email_is_unique', siteUrl('auth/password/forgotten')) - : null, - Config::get('APP_NATIVE_LOGIN_USING') === 'USERNAME' - ? lang('auth_register_username_is_unique', siteUrl('auth/password/forgotten')) - : null, - Config::get('APP_NATIVE_LOGIN_USING') === 'BOTH' - ? lang('auth_register_identity_is_unique', siteUrl('auth/password/forgotten')) - : null, - ]), - ]) - ->run(); + ->run($oInput->post()); if (appSetting('user_registration_captcha_enabled', 'auth')) { if (!$oCaptchaService->verify()) { diff --git a/src/Admin/Controller/Accounts.php b/src/Admin/Controller/Accounts.php index ad38f570..175b485b 100755 --- a/src/Admin/Controller/Accounts.php +++ b/src/Admin/Controller/Accounts.php @@ -22,6 +22,7 @@ use Nails\Auth\Model\User; use Nails\Auth\Model\User\Group; use Nails\Auth\Model\User\Password; +use Nails\Auth\Validator\User\Identity; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\ModelException; use Nails\Common\Exception\NailsException; @@ -363,38 +364,18 @@ public function create(): void if ($oInput->post()) { - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); - - $aRules = [ - 'group_id' => [FormValidation::RULE_REQUIRED, FormValidation::RULE_IS_NATURAL_NO_ZERO], - 'first_name' => [FormValidation::RULE_REQUIRED, FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150)], - 'last_name' => [FormValidation::RULE_REQUIRED, FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150)], - 'email' => [ - FormValidation::RULE_REQUIRED, - FormValidation::RULE_VALID_EMAIL, - FormValidation::rule(FormValidation::RULE_IS_UNIQUE, Config::get('NAILS_DB_PREFIX') . 'user_email', 'email'), - FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 255), - ], - ]; - - if (in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['BOTH', 'USERNAME'])) { - $aRules['username'] = [ - FormValidation::RULE_REQUIRED, - FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150), - FormValidation::RULE_ALPHA_DASH_PERIOD, - FormValidation::rule(FormValidation::RULE_IS_UNIQUE, Config::get('NAILS_DB_PREFIX') . 'user', 'username'), - ]; - } - try { - $oFormValidation - ->buildValidator($aRules, [ + (new Identity()) + ->addRules([ + 'group_id' => [FormValidation::RULE_REQUIRED, FormValidation::RULE_IS_NATURAL_NO_ZERO], + 'first_name' => [FormValidation::RULE_REQUIRED, FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150)], + 'last_name' => [FormValidation::RULE_REQUIRED, FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150)], + ]) + ->setMessages([ FormValidation::RULE_IS_NATURAL_NO_ZERO => lang('fv_required'), - FormValidation::RULE_IS_UNIQUE => lang('fv_email_already_registered'), ]) - ->run(); + ->run($oInput->post()); // Success diff --git a/src/Validator/User/Identifier.php b/src/Validator/User/Identifier.php new file mode 100644 index 00000000..d765b6cd --- /dev/null +++ b/src/Validator/User/Identifier.php @@ -0,0 +1,34 @@ + Config::get('APP_NATIVE_LOGIN_USING') === 'EMAIL' + ? [FormValidation::RULE_REQUIRED, FormValidation::RULE_VALID_EMAIL] + : [FormValidation::RULE_REQUIRED], + ]; + } +} diff --git a/src/Validator/User/Identity.php b/src/Validator/User/Identity.php new file mode 100644 index 00000000..f9d47f62 --- /dev/null +++ b/src/Validator/User/Identity.php @@ -0,0 +1,129 @@ +addRules(['first_name' => [FormValidation::RULE_REQUIRED]]) + * ->run($oInput->post()); + * + * Pass a user ID to exempt that user's own email/username from the uniqueness check + * when validating an edit. Override a field with `[]` to leave it unvalidated. + * + * @package Nails + * @subpackage module-auth + * @category Validator + * @author Nails Dev Team + * @link + */ + +namespace Nails\Auth\Validator\User; + +use Nails\Auth\Constants; +use Nails\Auth\Model\User; +use Nails\Common\Exception\FactoryException; +use Nails\Common\Factory\Service\FormValidation\Validator; +use Nails\Common\Service\FormValidation; +use Nails\Config; +use Nails\Factory; + +class Identity extends Validator +{ + /** + * @param int|null $iIgnoreUserId A user whose own email/username should not count as taken + */ + public function __construct(private readonly ?int $iIgnoreUserId = null) + { + parent::__construct(); + } + + // -------------------------------------------------------------------------- + + /** + * Whether the app identifies users by email + */ + public static function usesEmail(): bool + { + return in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['EMAIL', 'BOTH'], true); + } + + /** + * Whether the app identifies users by username + */ + public static function usesUsername(): bool + { + return in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['USERNAME', 'BOTH'], true); + } + + // -------------------------------------------------------------------------- + + /** + * @throws FactoryException + */ + protected function rules(): array + { + $aRules = []; + + if (static::usesEmail()) { + /** @var User\Email $oUserEmailModel */ + $oUserEmailModel = Factory::model('UserEmail', Constants::MODULE_SLUG); + + $aRules['email'] = [ + 'trim', + FormValidation::RULE_REQUIRED, + FormValidation::RULE_VALID_EMAIL, + FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 255), + $this->uniqueRule($oUserEmailModel->getTableName(), 'email', 'user_id'), + ]; + } + + if (static::usesUsername()) { + /** @var User $oUserModel */ + $oUserModel = Factory::model('User', Constants::MODULE_SLUG); + + $aRules['username'] = [ + 'trim', + FormValidation::RULE_REQUIRED, + FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150), + FormValidation::RULE_ALPHA_DASH_PERIOD, + $this->uniqueRule($oUserModel->getTableName(), 'username', 'id'), + ]; + } + + return $aRules; + } + + // -------------------------------------------------------------------------- + + protected function messages(): array + { + $sKey = match (Config::get('APP_NATIVE_LOGIN_USING')) { + 'EMAIL' => 'auth_register_email_is_unique', + 'USERNAME' => 'auth_register_username_is_unique', + default => 'auth_register_identity_is_unique', + }; + + return [ + FormValidation::RULE_IS_UNIQUE => lang($sKey, siteUrl('auth/password/forgotten')), + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Compiles an `is_unique` rule, exempting the ignored user if one was given + * + * @param string $sTable The table to check + * @param string $sColumn The column holding the value + * @param string $sUserIdColumn The column holding the user ID in that table + */ + protected function uniqueRule(string $sTable, string $sColumn, string $sUserIdColumn): string + { + return $this->iIgnoreUserId + ? FormValidation::rule(FormValidation::RULE_IS_UNIQUE, $sTable, $sColumn, $this->iIgnoreUserId, $sUserIdColumn) + : FormValidation::rule(FormValidation::RULE_IS_UNIQUE, $sTable, $sColumn); + } +} diff --git a/tests/Validator/User/IdentifierTest.php b/tests/Validator/User/IdentifierTest.php new file mode 100644 index 00000000..a484c913 --- /dev/null +++ b/tests/Validator/User/IdentifierTest.php @@ -0,0 +1,37 @@ +run($aData); + return []; + } catch (ValidationException $e) { + return $e->getData(); + } + } + + public function test_the_identifier_must_be_an_email_when_logging_in_by_email(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + self::assertSame(['identifier' => 'This must be a valid email.'], $this->errorsFor(['identifier' => 'ada'])); + self::assertSame([], $this->errorsFor(['identifier' => 'ada@example.com'])); + } + + public function test_any_value_will_do_otherwise(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'BOTH'); + + self::assertSame(['identifier' => 'This field is required.'], $this->errorsFor(['identifier' => ''])); + self::assertSame([], $this->errorsFor(['identifier' => 'ada'])); + } +} diff --git a/tests/Validator/User/IdentityTest.php b/tests/Validator/User/IdentityTest.php new file mode 100644 index 00000000..a505b6de --- /dev/null +++ b/tests/Validator/User/IdentityTest.php @@ -0,0 +1,123 @@ + taken values) + */ + private function validator(array $aTaken = [], ?int $iIgnoreUserId = null): Identity + { + return (new Identity($iIgnoreUserId)) + ->stubRule(FormValidation::RULE_IS_UNIQUE, function ($mValue, Context $oContext) use ($aTaken) { + [, $sColumn] = $oContext->getParams(); + return !in_array($mValue, $aTaken[$sColumn] ?? [], true); + }); + } + + private function errorsFor(Identity $oValidator, array $aData): array + { + try { + $oValidator->run($aData); + return []; + } catch (ValidationException $e) { + return $e->getData(); + } + } + + // -------------------------------------------------------------------------- + + public function test_both_fields_are_required_when_logging_in_with_both(): void + { + self::assertSame( + ['email', 'username'], + array_keys($this->errorsFor($this->validator(), [])) + ); + } + + public function test_a_clean_identity_passes_and_is_trimmed(): void + { + $oValidator = $this->validator(); + $oValidator->run(['email' => ' ada@example.com ', 'username' => ' ada.lovelace ']); + + self::assertSame( + ['email' => 'ada@example.com', 'username' => 'ada.lovelace'], + $oValidator->getValidatedData() + ); + } + + public function test_a_taken_identity_reports_the_already_registered_message(): void + { + $aErrors = $this->errorsFor( + $this->validator(['email' => ['ada@example.com']]), + ['email' => 'ada@example.com', 'username' => 'ada'] + ); + + self::assertSame(['email'], array_keys($aErrors)); + self::assertStringContainsString('already registered', $aErrors['email']); + } + + public function test_the_ignored_user_is_passed_to_the_uniqueness_rule(): void + { + $aSeen = []; + $oValidator = (new Identity(42)) + ->stubRule(FormValidation::RULE_IS_UNIQUE, function ($mValue, Context $oContext) use (&$aSeen) { + $aSeen[$oContext->getField()] = $oContext->getParams(); + return true; + }); + + $oValidator->run(['email' => 'ada@example.com', 'username' => 'ada']); + + self::assertSame(['42', 'user_id'], array_slice($aSeen['email'], 2)); + self::assertSame(['42', 'id'], array_slice($aSeen['username'], 2)); + } + + public function test_malformed_values_fail_before_uniqueness_is_checked(): void + { + $aErrors = $this->errorsFor( + $this->validator(), + ['email' => 'not-an-email', 'username' => 'has spaces!'] + ); + + self::assertSame('This must be a valid email.', $aErrors['email']); + self::assertStringContainsString('alpha-numeric', $aErrors['username']); + } + + public function test_only_the_configured_identity_is_validated(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + self::assertSame(['email'], array_keys($this->validator()->getRules())); + self::assertSame([], $this->errorsFor($this->validator(), ['email' => 'ada@example.com'])); + } + + public function test_a_field_can_be_switched_off_by_overriding_it_with_no_rules(): void + { + $oValidator = $this->validator()->setRules(['username' => []]); + + self::assertSame([], $this->errorsFor($oValidator, ['email' => 'ada@example.com'])); + } + + public function test_extra_rules_merge_with_the_identity_rules(): void + { + $oValidator = $this->validator()->addRules(['first_name' => [FormValidation::RULE_REQUIRED]]); + + self::assertSame( + ['email', 'username', 'first_name'], + array_keys($this->errorsFor($oValidator, [])) + ); + } +} From 28c5b58cb0ab52255d6b09ac52e6d0a8dcf158c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20de=20la=20Pen=CC=83a?= Date: Fri, 4 Sep 2026 09:45:51 +0100 Subject: [PATCH 5/7] refactor: Read the target user via `Context` in the merge validator The merge rule compared against `$oInput->post('user_id')`; it now reads the field from the data being validated via `Context::getValue()`. Co-Authored-By: Claude Fable 5.1 (cherry picked from commit 4b314132bd256cfb75bc314736f2875203cd7f73) --- src/Admin/Controller/Merge.php | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Admin/Controller/Merge.php b/src/Admin/Controller/Merge.php index dbe07950..f29228b6 100644 --- a/src/Admin/Controller/Merge.php +++ b/src/Admin/Controller/Merge.php @@ -22,6 +22,7 @@ use Nails\Common\Exception\ValidationException; use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; +use Nails\Common\Validation\Context; use Nails\Factory; use stdClass; @@ -88,12 +89,12 @@ public function index(): void ], 'merge_ids' => [ $oFormValidation::RULE_REQUIRED, - function ($mInput) use ($oInput) { + function ($mInput, Context $oContext) { $aMergeIds = explode(',', $mInput); if (in_array(activeUser('id'), $aMergeIds)) { throw new ValidationException('You cannot list yourself as a user to merge.'); - } elseif (in_array($oInput->post('user_id'), $aMergeIds)) { + } elseif (in_array($oContext->getValue('user_id'), $aMergeIds)) { throw new ValidationException('You cannot merge the target user into itself.'); } }, From 9ad8cc902ea662f12f23ca15793804c28bc7ca57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pablo=20de=20la=20Pen=CC=83a?= Date: Fri, 4 Sep 2026 10:10:05 +0100 Subject: [PATCH 6/7] chore: Boot Nails in the PHPUnit bootstrap `tests/bootstrap.php` now calls `Nails\Testing::bootstrapModule()` (and `phpunit.xml` supplies the `PRIVATE_KEY` it needs), matching `nails/common` and the other modules, so tests can use the Factory, services and models without CodeIgniter. The `Validator\User` tests depend on this and failed on CI with "No containers registered for nails/common". Co-Authored-By: Claude Fable 5.1 (cherry picked from commit be4baf0a00a5982f662f609fca201e8b780af998) --- phpunit.xml | 3 +++ tests/bootstrap.php | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/phpunit.xml b/phpunit.xml index fb22583f..d3c25537 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -4,4 +4,7 @@ ./tests + + + diff --git a/tests/bootstrap.php b/tests/bootstrap.php index fd349ea2..93ac78fe 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -1,3 +1,7 @@ Date: Sun, 6 Sep 2026 19:54:24 +0100 Subject: [PATCH 7/7] feat: Move user import processing to background LArge refactor to improve the handling of importing suers. Anything but modest CSVs would cause timeouts with no sane path to recovery or reporting. This commit overhauls this feature to add that, and more. (cherry picked from commit df55b337d3c93249b26cbe582ef65f53a318f941) --- .gitignore | 3 + admin/views/Import/index.php | 43 +- admin/views/Import/preview.php | 353 +++- assets/css/admin.min.css | 2 +- assets/js/admin.js | 6 + assets/js/components/UserImport.js | 1617 ++++++++++++++++ assets/sass/admin.scss | 144 ++ auth/config/email_types.php | 22 + auth/views/email/user_import_complete.php | 49 + .../email/user_import_complete_plaintext.php | 29 + auth/views/email/user_import_failed.php | 29 + .../email/user_import_failed_plaintext.php | 23 + composer.json | 13 +- services/services.php | 70 + src/Admin/Controller/Import.php | 888 +++++---- src/Api/Controller/Import.php | 440 +++++ src/Console/Command/User/Import/Clean.php | 261 +++ src/Console/Command/User/Import/Process.php | 312 +++ src/Cron/Task/User/Import/Clean.php | 39 + src/Cron/Task/User/Import/Process.php | 42 + src/Database/Migration/Migration20.php | 92 + src/Enum/User/Import/ItemStatus.php | 17 + src/Enum/User/Import/Runner.php | 16 + src/Enum/User/Import/Status.php | 127 ++ src/Exception/User/Import/LogException.php | 19 + .../User/Import/TemplateException.php | 19 + src/Factory/Email/User/Import/Complete.php | 51 + src/Factory/Email/User/Import/Failed.php | 51 + src/Model/User/Import.php | 312 +++ src/Model/User/Import/Item.php | 79 + src/Queue/Task/User/Import.php | 130 ++ src/Resource/User/Import.php | 125 ++ src/Resource/User/Import/Item.php | 33 + src/Service/User/Import.php | 238 ++- src/Service/User/Import/Csv.php | 358 ++++ src/Service/User/Import/Dispatcher.php | 122 ++ src/Service/User/Import/Processor.php | 1674 +++++++++++++++++ src/Service/User/Import/Validator.php | 394 ++++ tests/Enum/User/Import/StatusTest.php | 87 + tests/Model/User/ImportModelTest.php | 229 +++ tests/Resource/User/ImportTest.php | 134 ++ tests/Service/User/Import/CsvTest.php | 278 +++ tests/Service/User/Import/ProcessorTest.php | 1346 +++++++++++++ tests/Service/User/Import/ValidatorTest.php | 580 ++++++ tests/Service/User/ImportTest.php | 193 ++ tests/Stub/CdnSpy.php | 128 ++ tests/Stub/DatabaseSpy.php | 80 + tests/Stub/ImportModelRecorder.php | 113 ++ tests/Stub/ImportModelWithSpy.php | 34 + tests/Stub/ImportServiceStub.php | 193 ++ tests/Stub/LoggerSpy.php | 68 + tests/Stub/ProcessorWithSpies.php | 330 ++++ tests/Stub/UserModelStub.php | 67 + tests/Stub/ValidatorWithStub.php | 24 + 54 files changed, 11675 insertions(+), 451 deletions(-) create mode 100644 assets/js/components/UserImport.js create mode 100644 auth/views/email/user_import_complete.php create mode 100644 auth/views/email/user_import_complete_plaintext.php create mode 100644 auth/views/email/user_import_failed.php create mode 100644 auth/views/email/user_import_failed_plaintext.php create mode 100644 src/Api/Controller/Import.php create mode 100644 src/Console/Command/User/Import/Clean.php create mode 100644 src/Console/Command/User/Import/Process.php create mode 100644 src/Cron/Task/User/Import/Clean.php create mode 100644 src/Cron/Task/User/Import/Process.php create mode 100644 src/Database/Migration/Migration20.php create mode 100644 src/Enum/User/Import/ItemStatus.php create mode 100644 src/Enum/User/Import/Runner.php create mode 100644 src/Enum/User/Import/Status.php create mode 100644 src/Exception/User/Import/LogException.php create mode 100644 src/Exception/User/Import/TemplateException.php create mode 100644 src/Factory/Email/User/Import/Complete.php create mode 100644 src/Factory/Email/User/Import/Failed.php create mode 100644 src/Model/User/Import.php create mode 100644 src/Model/User/Import/Item.php create mode 100644 src/Queue/Task/User/Import.php create mode 100644 src/Resource/User/Import.php create mode 100644 src/Resource/User/Import/Item.php create mode 100644 src/Service/User/Import/Csv.php create mode 100644 src/Service/User/Import/Dispatcher.php create mode 100644 src/Service/User/Import/Processor.php create mode 100644 src/Service/User/Import/Validator.php create mode 100644 tests/Enum/User/Import/StatusTest.php create mode 100644 tests/Model/User/ImportModelTest.php create mode 100644 tests/Resource/User/ImportTest.php create mode 100644 tests/Service/User/Import/CsvTest.php create mode 100644 tests/Service/User/Import/ProcessorTest.php create mode 100644 tests/Service/User/Import/ValidatorTest.php create mode 100644 tests/Service/User/ImportTest.php create mode 100644 tests/Stub/CdnSpy.php create mode 100644 tests/Stub/DatabaseSpy.php create mode 100644 tests/Stub/ImportModelRecorder.php create mode 100644 tests/Stub/ImportModelWithSpy.php create mode 100644 tests/Stub/ImportServiceStub.php create mode 100644 tests/Stub/LoggerSpy.php create mode 100644 tests/Stub/ProcessorWithSpies.php create mode 100644 tests/Stub/UserModelStub.php create mode 100644 tests/Stub/ValidatorWithStub.php diff --git a/.gitignore b/.gitignore index 021a42aa..be155a64 100644 --- a/.gitignore +++ b/.gitignore @@ -248,3 +248,6 @@ $RECYCLE.BIN/ *.lnk # End of https://www.gitignore.io/api/linux,macos,windows,phpstorm,visualstudiocode,composer,node + +# PHPUnit +.phpunit.result.cache diff --git a/admin/views/Import/index.php b/admin/views/Import/index.php index 649c3beb..28184005 100644 --- a/admin/views/Import/index.php +++ b/admin/views/Import/index.php @@ -6,8 +6,24 @@ /** * @var array $additionalFields + * @var bool $bTemplateUnusable */ +/** + * The controller has already reported why; offering a form which cannot work + * would only invite an upload that is certain to be rejected. + */ +if (!empty($bTemplateUnusable)) { + ?> +
+

+ User import is unavailable until the template is corrected. +

+
+
@@ -29,6 +45,8 @@

Please note: The CSV you supply should be in the correct format, as per the template which you can download above. Remember to include the header rows describing each column. +
Every row is validated when you upload; if any of them cannot be imported the file is rejected + and nothing is created, so you can correct it and try again.

[ - 'text' => 'Preview', - 'name' => 'action', - 'value' => 'preview', + 'text' => 'Upload & Preview', ], ]); echo form_close(); ?> +
diff --git a/admin/views/Import/preview.php b/admin/views/Import/preview.php index 9e17a188..0a15304a 100644 --- a/admin/views/Import/preview.php +++ b/admin/views/Import/preview.php @@ -1,65 +1,336 @@ $aRegistered */ +$bIsDraft = $oImport->status === Status::DRAFT; +$iColumns = count($aKeys) + 1; +$iRowCount = (int) ($oImport->row_count ?? 0); + +/** + * Rows whose email or username already belongs to an account. The CSV is not + * wrong - these rows are simply redundant - so rather than rejecting the file + * the admin is offered the chance to import the rest without them. + */ +$aRegistered = $aRegistered ?? []; +$iRegistered = count($aRegistered); +$iImportable = max(0, $iRowCount - $iRegistered); + +/** + * The confirmation can be exact rather than hedging with "up to": both routes + * to it - ticking the skip checkbox below, or Continue on the warning an + * unticked Import opens - agree to skip exactly these rows. + */ +$sConfirmBody = sprintf( + '%s %s will be created in the background%s. This cannot be undone.', + number_format($iImportable), + $iImportable === 1 ? 'user account' : 'user accounts', + $iRegistered + ? sprintf( + ', and %s %s will be skipped', + number_format($iRegistered), + $iRegistered === 1 ? 'row' : 'rows' + ) + : '' +); + +/** + * The warning an unticked Import opens, reusing the alert's own phrasing so the + * modal does not introduce a second account of the same problem. Continue is an + * implied tick, so the copy names both halves of what it agrees to. + */ +$sWarnTitle = sprintf( + '%s %s already registered', + number_format($iRegistered), + $iRegistered === 1 ? 'row is' : 'rows are' +); + +$sWarnBody = sprintf( + 'An account already exists for %s of the rows in this CSV, so %s cannot be imported. ' + . 'Continue to import the remaining %s and skip %s, or cancel and correct your CSV.', + number_format($iRegistered), + $iRegistered === 1 ? 'it' : 'they', + number_format($iImportable), + $iRegistered === 1 ? 'it' : 'them' +); + +/** + * Delete is a plain button which UserImport turns into a + * DELETE /api/auth/import/{id}; see \Nails\Auth\Api\Controller\Import. Unlike + * the approve form it does not degrade - with the bundle broken it does nothing + * - which is the same bargain the preview table below already makes. + * + * `type="button"` is load bearing: on a draft this sits inside the approve + * form, and the default `submit` would start the import. + * + * The status and the created count are handed over so that the confirmation + * copy can be composed in one place, in JS, rather than written out again here. + */ +/** + * Details is a plain button which UserImport turns into a modal; see + * assets/js/components/UserImport.js. Offered only when there is something to + * show - a stored reason, or rows which failed or warned - so a clean import + * does not get a button which opens an empty box. + */ +$bHasDetails = $oImport->status->isTerminal() + && ($oImport->error || $oImport->error_count || $oImport->warning_count); + +$sDetailsButton = $bHasDetails + ? sprintf( + '', + $oImport->id + ) + : null; + +$sDeleteButton = $oImport->status->isDeletable() + ? sprintf( + '', + implode(' ', [ + 'data-import-id="' . $oImport->id . '"', + 'data-delete-url="' . siteUrl('api/auth/import/' . $oImport->id) . '"', + 'data-redirect="' . siteUrl($sListUrl) . '"', + 'data-status="' . $oImport->status->value . '"', + // Warned rows have accounts too; see Enum\User\Import\ItemStatus + 'data-created-count="' . ($oImport->success_count + $oImport->warning_count) . '"', + ]) + ) + : null; + +/** + * A draft can be imported or deleted; anything else which is deletable gets + * Delete alone, with nothing to save. A job a runner holds gets no controls at + * all, so the bar is absent entirely for those two statuses. + */ +if ($bIsDraft) { + $aControls = [ + 'save' => ['text' => 'Import'], + 'html' => ['right' => $sDeleteButton], + ]; +} elseif ($sDeleteButton || $sDetailsButton) { + $aControls = [ + 'save' => ['enabled' => false], + 'html' => ['right' => trim($sDetailsButton . ' ' . $sDeleteButton)], + ]; +} else { + $aControls = null; +} + +/** + * Mirrors how module-admin normalises a column label into a cell class; see + * module-admin/admin/views/DefaultController/index.php + */ +$fFieldClass = function (string $sLabel): string { + $sLabel = strtolower($sLabel); + $sLabel = preg_replace('/[^a-z0-9 \-_]/', '', $sLabel); + return 'field field--' . str_replace([' ', '_'], '-', $sLabel); +}; + ?>
-
- Please review the following data -
Your CSV has been processed, and the following values have been ascertained. Please verify them, and when - happy to continue, click "Import" below. -
- - - - - - - - + +
+ Please review the following data +
Your CSV has been uploaded and every row has been validated. Please verify the values below, and + when happy to continue, click "Import" below. The import runs in the background; you can leave this page + once it has started. +
+ +
+ + + already registered + +
An account already exists for the following, so + cannot be imported. Tick the box below to import the + remaining , or correct your CSV and upload it again. +
$aLineErrors) { + foreach ($aLineErrors as $sLineError) { + echo htmlspecialchars( + sprintf('Line %d: %s', $iLine, $sLineError), + ENT_QUOTES + ) . '
'; + } + } + + ?> +
+ -
- - - +

+ +

+ '; - foreach ($aKeys as $sKey) { + ?> + +
+ This import is status->value?> and can no longer be changed. + error) { ?> -
+
error, 2)[0], ENT_QUOTES)?> '; + ?> +

+ + + +
+ > +
+ + + -
-
—'?>
+
+
+ + + + + + + + + + + + + + +
Line
+ + + Loading + +
+
+
+ [ - 'text' => 'Import', - 'name' => 'action', - 'value' => 'import', - ], - ]); + if ($aControls) { + echo Helper::floatingControls($aControls); + } + + if ($bIsDraft) { + echo form_close(); + } ?> - diff --git a/assets/css/admin.min.css b/assets/css/admin.min.css index 961d2d50..d87c9558 100644 --- a/assets/css/admin.min.css +++ b/assets/css/admin.min.css @@ -1 +1 @@ -.group-accounts.all table{margin-bottom:1em}.group-accounts.all table th.id,.group-accounts.all table th.group,.group-accounts.all table td.id,.group-accounts.all table td.group{text-align:center}.group-accounts.all table th.id,.group-accounts.all table td.id{width:50px;min-width:50px}.group-accounts.all table th.group,.group-accounts.all table td.group{width:150px;min-width:150px}.group-accounts.all table th.actions,.group-accounts.all table td.actions{width:225px;min-width:225px;text-align:center}.group-accounts.all table td.details img.profile-img{float:left;margin-right:10px;width:65px;height:65px;background:#ccc;border:1px solid #ccc;border-radius:2px}.group-accounts.all table td.details img.verified{margin-left:5px;width:10px;height:10px;position:relative;top:2px}.group-accounts.all table td.id,.group-accounts.all table td.group{vertical-align:middle;color:#666}.group-accounts.all table td.profile.zero{color:#ccc}.group-accounts.all table td.actions .not-editable{color:#aaa}.group-accounts.all table td .awesome{margin-right:5px;margin-bottom:5px;padding:8px 5px 6px 5px}.group-accounts.all table td div{float:left}.group-accounts.all table td div small{display:block}.group-accounts.create #user-group-descriptions li,.group-accounts.create #user-group-pwrules li{margin:0}.group-accounts.edit .social .icon{float:left;border:1px solid #999;border-radius:3px;background:#f3f3f3;padding:5px;margin-right:15px;padding-left:42px;line-height:16px;background-position:5px center;background-repeat:no-repeat;word-wrap:break-word}.group-accounts.edit .social p{margin-top:5px}.group-accounts.edit .uploads ul{max-height:500px;overflow:auto}.group-accounts.edit .uploads ul li{vertical-align:middle;display:block;margin-bottom:0}.group-accounts.edit .uploads ul li a{padding:1em;display:block;position:relative;min-height:51px;background:#efefef;background-clip:padding-box;text-decoration:none}.group-accounts.edit .uploads ul li a small{display:block}.group-accounts.edit .uploads ul li a.image{padding-left:55px}.group-accounts.edit .uploads ul li a.image img{position:absolute;left:5px;top:5px;width:35px;height:35px;border:1px solid #ccc;margin:0;margin-right:10px;vertical-align:middle;padding:2px;background:#fff;border-radius:2px}.group-accounts.edit .uploads ul li.no-data{color:#777}.group-accounts.edit table.emails th.email input,.group-accounts.edit table.emails td.email input{width:100%;box-sizing:border-box}.group-accounts.edit table.emails th.actions,.group-accounts.edit table.emails td.actions{width:220px}.group-accounts.groups table th,.group-accounts.groups table td{vertical-align:middle}.group-accounts.groups table th.actions,.group-accounts.groups table th.actions,.group-accounts.groups table td.actions,.group-accounts.groups table td.actions{min-width:220px;text-align:center}.group-accounts.groups table th.default,.group-accounts.groups table th.default,.group-accounts.groups table td.default,.group-accounts.groups table td.default{width:50px;text-align:center}.group-accounts.groups table th.default span,.group-accounts.groups table th.default span,.group-accounts.groups table td.default span,.group-accounts.groups table td.default span{font-size:1.5em}.group-accounts.groups.edit .permission-groups .permission-group th.enabled{width:50px}.group-accounts.groups.edit .permission-groups .permission-group td{cursor:pointer}.group-accounts.groups.edit .permission-groups .permission-group td.enabled{padding:0}.group-accounts.groups.edit .permission-groups .permission-group td.enabled label{width:100%;height:100%;cursor:pointer;line-height:40px}.group-accounts.groups.edit .search{border:none;border-bottom:1px solid #efefef}.group-accounts.groups.edit .search .search-text{margin:0}.group-accounts.merge.merge-preview table{margin-bottom:1.5em}.group-accounts.merge.merge-preview table th.userId,.group-accounts.merge.merge-preview table td.userId{text-align:center;width:50px}.group-accounts.merge.merge-preview table th.user-cell,.group-accounts.merge.merge-preview table td.user-cell{width:auto}.group-accounts.merge.merge-preview table th.tableRows,.group-accounts.merge.merge-preview table td.tableRows{text-align:center;width:100px}.group-accounts.change-group table th.userId,.group-accounts.change-group table td.userId{text-align:center;width:50px} +.group-accounts.all table{margin-bottom:1em}.group-accounts.all table th.id,.group-accounts.all table th.group,.group-accounts.all table td.id,.group-accounts.all table td.group{text-align:center}.group-accounts.all table th.id,.group-accounts.all table td.id{width:50px;min-width:50px}.group-accounts.all table th.group,.group-accounts.all table td.group{width:150px;min-width:150px}.group-accounts.all table th.actions,.group-accounts.all table td.actions{width:225px;min-width:225px;text-align:center}.group-accounts.all table td.details img.profile-img{float:left;margin-right:10px;width:65px;height:65px;background:#ccc;border:1px solid #ccc;border-radius:2px}.group-accounts.all table td.details img.verified{margin-left:5px;width:10px;height:10px;position:relative;top:2px}.group-accounts.all table td.id,.group-accounts.all table td.group{vertical-align:middle;color:#666}.group-accounts.all table td.profile.zero{color:#ccc}.group-accounts.all table td.actions .not-editable{color:#aaa}.group-accounts.all table td .awesome{margin-right:5px;margin-bottom:5px;padding:8px 5px 6px 5px}.group-accounts.all table td div{float:left}.group-accounts.all table td div small{display:block}.group-accounts.create #user-group-descriptions li,.group-accounts.create #user-group-pwrules li{margin:0}.group-accounts.edit .social .icon{float:left;border:1px solid #999;border-radius:3px;background:#f3f3f3;padding:5px;margin-right:15px;padding-left:42px;line-height:16px;background-position:5px center;background-repeat:no-repeat;word-wrap:break-word}.group-accounts.edit .social p{margin-top:5px}.group-accounts.edit .uploads ul{max-height:500px;overflow:auto}.group-accounts.edit .uploads ul li{vertical-align:middle;display:block;margin-bottom:0}.group-accounts.edit .uploads ul li a{padding:1em;display:block;position:relative;min-height:51px;background:#efefef;background-clip:padding-box;text-decoration:none}.group-accounts.edit .uploads ul li a small{display:block}.group-accounts.edit .uploads ul li a.image{padding-left:55px}.group-accounts.edit .uploads ul li a.image img{position:absolute;left:5px;top:5px;width:35px;height:35px;border:1px solid #ccc;margin:0;margin-right:10px;vertical-align:middle;padding:2px;background:#fff;border-radius:2px}.group-accounts.edit .uploads ul li.no-data{color:#777}.group-accounts.edit table.emails th.email input,.group-accounts.edit table.emails td.email input{width:100%;box-sizing:border-box}.group-accounts.edit table.emails th.actions,.group-accounts.edit table.emails td.actions{width:220px}.group-accounts.groups table th,.group-accounts.groups table td{vertical-align:middle}.group-accounts.groups table th.actions,.group-accounts.groups table th.actions,.group-accounts.groups table td.actions,.group-accounts.groups table td.actions{min-width:220px;text-align:center}.group-accounts.groups table th.default,.group-accounts.groups table th.default,.group-accounts.groups table td.default,.group-accounts.groups table td.default{width:50px;text-align:center}.group-accounts.groups table th.default span,.group-accounts.groups table th.default span,.group-accounts.groups table td.default span,.group-accounts.groups table td.default span{font-size:1.5em}.group-accounts.groups.edit .permission-groups .permission-group th.enabled{width:50px}.group-accounts.groups.edit .permission-groups .permission-group td{cursor:pointer}.group-accounts.groups.edit .permission-groups .permission-group td.enabled{padding:0}.group-accounts.groups.edit .permission-groups .permission-group td.enabled label{width:100%;height:100%;cursor:pointer;line-height:40px}.group-accounts.groups.edit .search{border:none;border-bottom:1px solid #efefef}.group-accounts.groups.edit .search .search-text{margin:0}.group-accounts.merge.merge-preview table{margin-bottom:1.5em}.group-accounts.merge.merge-preview table th.userId,.group-accounts.merge.merge-preview table td.userId{text-align:center;width:50px}.group-accounts.merge.merge-preview table th.user-cell,.group-accounts.merge.merge-preview table td.user-cell{width:auto}.group-accounts.merge.merge-preview table th.tableRows,.group-accounts.merge.merge-preview table td.tableRows{text-align:center;width:100px}.group-accounts.change-group table th.userId,.group-accounts.change-group table td.userId{text-align:center;width:50px}.badge.badge--default{background-color:#555}.badge.badge--info{background-color:#31708f}.badge.badge--success{background-color:#3c763d}.badge.badge--warning{background-color:#8a6d3b}.badge.badge--danger{background-color:#a94442}@-webkit-keyframes user-import-spin{to{transform:rotate(360deg)}}@keyframes user-import-spin{to{transform:rotate(360deg)}}.user-import-spinner{display:inline-block;width:.85em;height:.85em;vertical-align:-0.1em;border:2px solid currentColor;border-right-color:rgba(0,0,0,0);border-radius:50%;-webkit-animation:user-import-spin .75s linear infinite;animation:user-import-spin .75s linear infinite}.user-import-details p{margin:0 0 .5rem}.user-import-details__log,.user-import-details__rows{max-height:40vh;overflow:auto;margin:0 0 .75rem;padding:.75rem;font-size:12px;line-height:1.5;white-space:pre-wrap;word-break:break-word;background-color:#f5f5f5;border:1px solid #d9d9d9;border-radius:4px}.module-auth.import th.actions,.module-auth.import td.actions{width:275px;min-width:275px}.module-auth.import--preview #user-import-preview tbody{transition:opacity 150ms ease-in-out}.module-auth.import--preview #user-import-preview[aria-busy=true] tbody{opacity:.4}.module-auth.import--preview #user-import-preview[aria-busy=true] .pagination a{pointer-events:none}.module-auth.import--preview th.field--line,.module-auth.import--preview td.field--line{width:60px;text-align:center} diff --git a/assets/js/admin.js b/assets/js/admin.js index ac26b93c..a49f283d 100644 --- a/assets/js/admin.js +++ b/assets/js/admin.js @@ -6,6 +6,7 @@ import AccountEdit from './components/AccountEdit.js'; import AccountMerge from './components/AccountMerge.js'; import Groups from './components/Groups.js'; import SearchUser from './components/SearchUser.js'; +import UserImport from './components/UserImport.js'; (function() { window.NAILS.ADMIN.registerPlugin( @@ -33,5 +34,10 @@ import SearchUser from './components/SearchUser.js'; 'SearchUser', new SearchUser(window.NAILS.ADMIN) ); + window.NAILS.ADMIN.registerPlugin( + 'nails/module-auth', + 'UserImport', + new UserImport(window.NAILS.ADMIN) + ); })(); diff --git a/assets/js/components/UserImport.js b/assets/js/components/UserImport.js new file mode 100644 index 00000000..4afbe942 --- /dev/null +++ b/assets/js/components/UserImport.js @@ -0,0 +1,1617 @@ +/** + * Drives the two pieces of the user import UI which need live data: the list of + * recent imports on the index, and the paged preview of an uploaded CSV. + * + * The admin bundle is loaded on every admin page, so each half no-ops unless its + * container is in the DOM. + */ +class UserImport { + + /** + * Construct UserImport + * @param {Object} adminController The admin controller + */ + constructor(adminController) { + + this.adminController = adminController; + + /** + * How often, in ms, to poll while something might change + * @type {Number} + */ + this.pollActive = 2500; + + /** + * How often, in ms, to poll when everything has settled + * @type {Number} + */ + this.pollIdle = 30000; + + /** + * How many page links either side of the current page the paginator + * shows; matches the `num_links` the admin pagination partial configures + * @type {Number} + */ + this.numLinks = 5; + + this.endpoint = window.SITE_URL + 'api/auth/import'; + + this.dom = { + list: document.getElementById('user-import-list'), + listBody: document.getElementById('user-import-list-body'), + preview: document.getElementById('user-import-preview'), + previewBody: document.getElementById('user-import-preview-body') + }; + + this.imports = []; + this.listTimeout = null; + + /** + * The shared modal, created on first use. Null means "not yet asked for", + * false means "asked for and unavailable". + * @type {Object|Boolean|null} + */ + this.modal = null; + + /** + * Null until the first page of the preview has loaded; this is what + * distinguishes the first load from a page change + * @type {Number|null} + */ + this.previewPage = null; + + /** + * The IDs of imports whose delete request is in flight, consulted when + * the actions cell renders so that the button's busy state survives a + * poll landing mid-request + * @type {Set} + */ + this.deleting = new Set(); + + /** + * Whether a confirmation is on screen. The modal is shared, so a second + * confirm() would reassign its onHide and leave the first promise + * unsettled. + * @type {Boolean} + */ + this.confirming = false; + + this.bindConfirmForms(); + this.bindDelete(); + this.bindDetails(); + + if (this.dom.list && this.dom.listBody) { + this.adminController.log('Constructing user import list'); + this.loadList(); + } + + if (this.dom.preview && this.dom.previewBody) { + this.adminController.log('Constructing user import preview'); + this.bindPreview(); + + /** + * When the CSV has gone the server has already said so in the table, + * and the API could only 404; asking would just stack a modal on top + * of the flash error which is already on the page. + */ + if (!this.dom.preview.dataset.csvMissing) { + this.loadPreview(1); + } + } + } + + // -------------------------------------------------------------------------- + + /** + * Fetches a URL and returns the decoded API envelope + * + * On failure the rejection carries the response's status, and the API's own + * message where it got far enough to render one: the poll only needs to + * know that something went wrong, but a delete the user asked for has to be + * able to say what. + * + * @param {String} url The URL to fetch + * @param {Object} options Any fetch options to apply, e.g. {method: 'DELETE'} + * @return {Promise} Resolves with the parsed response body + */ + request(url, options) { + + let settings = Object.assign({method: 'GET', credentials: 'same-origin'}, options || {}); + + // Merged over the caller's, so a caller cannot drop what the API needs + settings.headers = Object.assign({}, settings.headers, { + 'Accept': 'application/json', + 'X-Requested-With': 'XMLHttpRequest' + }); + + return fetch(url, settings) + .then((response) => response.text().then((body) => { + + let payload = null; + + /** + * In production an unhandled exception is re-thrown to the + * global error handler, so a 500 arrives as an HTML page rather + * than an envelope; the status is then all we have to go on. + */ + try { + payload = body ? JSON.parse(body) : null; + } catch (e) { + payload = null; + } + + if (!response.ok) { + let error = new Error( + (payload && payload.error) || `Request failed with status ${response.status}` + ); + error.status = response.status; + throw error; + } + + return payload || {}; + })); + } + + // -------------------------------------------------------------------------- + + /** + * Loads, and schedules the next load of, the list of recent imports + * @return {void} + */ + loadList() { + + /** + * The preview page runs the same class but has no list, and the delete + * handlers resync unconditionally; without this a failed delete there + * would reach renderList() and fall over on a null container. + */ + if (!this.dom.list || !this.dom.listBody) { + return; + } + + clearTimeout(this.listTimeout); + + this.request(this.endpoint) + .then((response) => { + + let imports = response.data || []; + + this.renderList(imports); + + // Poll quickly while something is in flight, slowly otherwise + let isBusy = imports.some((item) => { + return ['PENDING', 'VALIDATING', 'RUNNING'].indexOf(item.status) !== -1; + }); + + this.listTimeout = setTimeout(() => { + this.loadList(); + }, isBusy ? this.pollActive : this.pollIdle); + }) + .catch((error) => { + + /** + * This path polls every few seconds, so it must never raise a + * modal; one flaky response would otherwise become an unclosable + * stream of dialogs. The next poll will recover on its own. + */ + this.adminController.warn('Failed to load user imports', error); + + this.listTimeout = setTimeout(() => { + this.loadList(); + }, this.pollIdle); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Reconciles the rendered rows against the response + * @param {Array} imports The imports as returned by the API + * @return {void} + */ + renderList(imports) { + + let responseIds = imports.map((item) => item.id); + + for (let i = 0; i < imports.length; i++) { + let existing = this.imports.find((x) => x.id === imports[i].id); + if (existing) { + /** + * The stored item is what the delete confirmation reads its + * status and counts from, so it has to keep up; `dom` survives + * because the response never carries one. + */ + Object.assign(existing, imports[i]); + this.updateRow(existing.dom, imports[i]); + } else { + this.addRow(imports[i]); + } + } + + // Reverse loop so items can be spliced out as we go + for (let i = this.imports.length - 1; i >= 0; i--) { + if (responseIds.indexOf(this.imports[i].id) === -1) { + if (this.imports[i].dom && this.imports[i].dom.parentNode) { + this.imports[i].dom.parentNode.removeChild(this.imports[i].dom); + } + this.imports.splice(i, 1); + } + } + + /** + * Revealed on the first successful response and never hidden again, so an + * admin with no imports sees the empty state rather than nothing at all. + */ + this.dom.list.classList.remove('hidden'); + this.toggleListEmptyRow(); + } + + // -------------------------------------------------------------------------- + + /** + * Adds or removes the list's empty state row + * @return {void} + */ + toggleListEmptyRow() { + + let row = this.dom.listBody.querySelector('tr.user-import-list-empty'); + + if (this.imports.length) { + if (row) { + row.parentNode.removeChild(row); + } + return; + } + + if (!row) { + row = document.createElement('tr'); + row.classList.add('user-import-list-empty'); + row.appendChild(this.createCell( + 'no-data', + 'No imports found', + this.countColumns(this.dom.list) + )); + this.dom.listBody.appendChild(row); + } + } + + // -------------------------------------------------------------------------- + + /** + * Drops an import from the list, and from the reconciliation set + * + * Splicing `imports` is the point: leave the entry behind and the next + * poll's find() matches it, updateRow() writes into a detached node, the + * row never comes back, and the empty state never appears either. + * + * @param {Number} id The import to remove + * @return {void} + */ + removeImport(id) { + + let index = this.imports.findIndex((x) => x.id === id); + if (index === -1) { + return; + } + + let row = this.imports[index].dom; + if (row && row.parentNode) { + row.parentNode.removeChild(row); + } + + this.imports.splice(index, 1); + this.toggleListEmptyRow(); + } + + // -------------------------------------------------------------------------- + + /** + * Adds a row for an import which was not previously rendered + * @param {Object} item The import + * @return {void} + */ + addRow(item) { + + let tr = document.createElement('tr'); + + tr.appendChild(this.createCell(['field', 'field--id'], this.getIdCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--file'], this.getFileCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--status'], this.getStatusCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--progress'], this.getProgressCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--requested', 'datetime'], this.getRequestedCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--finished', 'datetime'], this.getFinishedCellHtml(item))); + + /** + * Left genuinely empty when there is nothing to do, so that admin's + * `td.actions:empty:before` supplies the "No Actions" label + */ + tr.appendChild(this.createCell('actions', this.getActionsCellHtml(item))); + + item.dom = tr; + this.imports.push(item); + this.dom.listBody.appendChild(tr); + } + + // -------------------------------------------------------------------------- + + /** + * Updates the cells of an already rendered import + * @param {Object} dom The row element + * @param {Object} item The import + * @return {void} + */ + updateRow(dom, item) { + + this.setCellHtml(dom, 'td.field--status', this.getStatusCellHtml(item)); + this.setCellHtml(dom, 'td.field--progress', this.getProgressCellHtml(item)); + this.setCellHtml(dom, 'td.field--finished', this.getFinishedCellHtml(item)); + this.setCellHtml(dom, 'td.actions', this.getActionsCellHtml(item)); + } + + // -------------------------------------------------------------------------- + + /** + * Sets a cell's contents, if the cell exists + * + * Guarded because this runs inside the poll loop; a renamed class should cost + * one stale cell, not a stream of errors every few seconds. + * + * @param {HTMLElement} row The row + * @param {String} selector The cell's selector + * @param {String} html The cell's contents + * @return {void} + */ + setCellHtml(row, selector, html) { + let cell = row.querySelector(selector); + if (cell) { + cell.innerHTML = html; + } + } + + // -------------------------------------------------------------------------- + + /** + * The ID cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getIdCellHtml(item) { + return this.escape(item.id); + } + + // -------------------------------------------------------------------------- + + /** + * The file cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getFileCellHtml(item) { + let name = item.source && item.source.filename ? item.source.filename : 'CSV'; + return `${this.escape(name)}`; + } + + // -------------------------------------------------------------------------- + + /** + * The status cell's contents + * + * The badge carries the colour; the cell itself is left untinted, so a row + * no longer changes colour wholesale. admin's `td small { display: block }` + * puts the error underneath without any help from us. + * + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getStatusCellHtml(item) { + + let html = `` + + this.escape(item.status) + + ''; + + /** + * The summary line only - the stored error is deliberately multi-line + * (see Processor::composeError()) and the whole thing here would turn + * every failed row into a wall of text. The Details button has the rest. + */ + if (item.error) { + html += `${this.escape(item.error_summary || item.error)}`; + } + + return html; + } + + // -------------------------------------------------------------------------- + + /** + * The badge modifier for an import's status + * + * Mapped rather than derived: the return goes straight into a `class` + * attribute, and a lookup is what guarantees it can only ever be one of the + * five modifiers the stylesheet declares - including for a status case + * shipped after this bundle was last built. + * + * @param {Object} item The import + * @return {String} The modifier class + */ + getStatusVariant(item) { + return { + DRAFT: 'badge--default', + PENDING: 'badge--info', + VALIDATING: 'badge--info', + RUNNING: 'badge--info', + COMPLETE: 'badge--success', + PARTIAL: 'badge--warning', + FAILED: 'badge--danger' + }[item.status] || 'badge--default'; + } + + // -------------------------------------------------------------------------- + + /** + * The progress cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getProgressCellHtml(item) { + + let progress = item.progress || {}; + let total = progress.row_count || 0; + + if (!total) { + return '—'; + } + + let cursor = item.status === 'VALIDATING' + ? progress.validated_count + : progress.processed_count; + + let html = `${this.numberFormat(cursor)} of ${this.numberFormat(total)} (${progress.percent}%)`; + + /** + * Warnings get their own line rather than being folded into the errors: + * an errored row has no account and a warned one does, so they are not + * the same number to anybody reading this. + */ + if (progress.error_count) { + html += `${this.numberFormat(progress.error_count)} errored`; + } + + if (progress.warning_count) { + html += `${this.numberFormat(progress.warning_count)} warned`; + } + + return html; + } + + // -------------------------------------------------------------------------- + + /** + * The requested cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getRequestedCellHtml(item) { + let when = item.created ? item.created.formatted : ''; + let who = item.user ? item.user.name : null; + return who ? `${when}${this.escape(who)}` : when; + } + + // -------------------------------------------------------------------------- + + /** + * The finished cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getFinishedCellHtml(item) { + return item.finished + ? item.finished.formatted + : '—'; + } + + // -------------------------------------------------------------------------- + + /** + * The actions cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getActionsCellHtml(item) { + + let buttons = []; + + if (item.status === 'DRAFT') { + buttons.push(`Review`); + } + + if (this.hasDetails(item)) { + buttons.push('`); + } + + if (item.log && item.log.url) { + buttons.push(`Log`); + } + + if (this.isDeletable(item)) { + /** + * Rendered busy while the request is in flight, because the poll + * re-renders this cell out from under the button - every 2.5s while + * something is active, and every 30s when nothing is, which is the + * case that actually bites. + */ + buttons.push(this.deleting.has(item.id) + ? '' + : '` + ); + } + + return buttons.join(' '); + } + + // -------------------------------------------------------------------------- + + /** + * Whether an import can be deleted + * + * Mirrors Status::isDeletable(). The server checks again and answers 409 if + * this is out of date, which it will be for up to one poll. + * + * @param {Object} item The import + * @return {Boolean} Whether it can be deleted + */ + isDeletable(item) { + return ['VALIDATING', 'RUNNING'].indexOf(item.status) === -1; + } + + // -------------------------------------------------------------------------- + + /** + * Whether an import has anything worth opening a modal for + * + * Either a stored reason, or per-row failures or warnings to list. A clean + * import has none of them and gets no button, which is what keeps the list + * uncluttered. + * + * @param {Object} item The import + * @return {Boolean} Whether there is detail to show + */ + hasDetails(item) { + let progress = item.progress || {}; + return Boolean(item.error) || + Boolean(progress.error_count) || + Boolean(progress.warning_count); + } + + // -------------------------------------------------------------------------- + + /** + * Wires up the preview's paging + * @return {void} + */ + bindPreview() { + + this.dom.previewPagingTop = this.dom.preview.querySelector('[data-paging="top"]'); + this.dom.previewPagingBottom = this.dom.preview.querySelector('[data-paging="bottom"]'); + + /** + * Delegated, because both paginators are re-rendered on every response; + * listeners bound to the buttons themselves would go stale immediately. + */ + this.dom.preview + .addEventListener('click', (event) => { + + let link = event.target.closest('a[data-page]'); + if (!link) { + return; + } + + event.preventDefault(); + + // Anchors cannot be disabled, so the busy state is the guard + if (this.dom.preview.getAttribute('aria-busy') === 'true') { + return; + } + + this.loadPreview(parseInt(link.dataset.page, 10), link); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Loads a page of the CSV preview + * @param {Number} page The page to load + * @param {HTMLElement} trigger The button which asked for it, if any + * @return {void} + */ + loadPreview(page, trigger) { + + let id = this.dom.preview.dataset.importId; + let isFirstLoad = this.previewPage === null; + + /** + * On the first load the server has already rendered a spinner row, so + * there is nothing to dim and no button to put a spinner on. + */ + if (!isFirstLoad) { + this.setPreviewBusy(true, trigger); + } + + this.request(`${this.endpoint}/${id}/rows?page=${page}`) + .then((response) => { + + this.previewPage = page; + this.renderPreview(response.data || [], response.meta || {}); + + // The first load is already at the top of the page; only a page change moves + if (!isFirstLoad) { + this.scrollToPreview(); + } + }) + .catch((error) => { + + this.adminController.warn('Failed to load user import preview', error); + + /** + * On a page change the rows on screen are still valid, so they + * stay; losing what you were reading because page seven timed out + * is worse than the error itself. + */ + if (isFirstLoad) { + this.renderPreviewMessage('The preview could not be loaded'); + } + + this.alert( + 'The preview could not be loaded', + 'Something went wrong fetching this page of the CSV. Please reload the page to try again.' + ); + }) + .finally(() => { + this.setPreviewBusy(false); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Toggles the preview's busy state + * + * `aria-busy` doubles as the styling hook, so no class has to be invented; it + * is also what stops the paging links being followed, since an anchor has no + * disabled state. + * + * @param {Boolean} isBusy Whether a request is in flight + * @param {HTMLElement} trigger The link which asked for it, if any + * @return {void} + */ + setPreviewBusy(isBusy, trigger) { + + this.dom.preview.setAttribute('aria-busy', isBusy ? 'true' : 'false'); + + if (isBusy) { + + if (trigger) { + this.previewTrigger = {el: trigger, label: trigger.innerHTML}; + trigger.innerHTML = ''; + } + + } else if (this.previewTrigger) { + + /** + * A successful load re-renders the paginator, taking the link with + * it; the link is only still around when the request failed, and then + * it must not be left spinning. + */ + if (this.previewTrigger.el.isConnected) { + this.previewTrigger.el.innerHTML = this.previewTrigger.label; + } + + this.previewTrigger = null; + } + } + + // -------------------------------------------------------------------------- + + /** + * Renders a page of the CSV preview + * @param {Array} rows The rows to render + * @param {Object} meta The response's meta data + * @return {void} + */ + renderPreview(rows, meta) { + + let header = meta.header || []; + let pagination = meta.pagination || {}; + + if (!rows.length) { + this.renderPreviewMessage('There is nothing to preview'); + + } else { + + this.dom.previewBody.innerHTML = ''; + + for (let i = 0; i < rows.length; i++) { + + let tr = document.createElement('tr'); + tr.appendChild(this.createCell(['field', 'field--line'], String(rows[i].line))); + + for (let j = 0; j < header.length; j++) { + let value = rows[i].data[header[j]]; + tr.appendChild(this.createCell( + this.fieldClass(header[j]).split(' '), + value === null || value === '' || typeof value === 'undefined' + ? '—' + : this.escape(String(value)) + )); + } + + this.dom.previewBody.appendChild(tr); + } + } + + this.renderPaginator(this.dom.previewPagingTop, pagination); + this.renderPaginator(this.dom.previewPagingBottom, pagination); + } + + // -------------------------------------------------------------------------- + + /** + * Replaces the preview's rows with a single full width message + * @param {String} message The message to show + * @return {void} + */ + renderPreviewMessage(message) { + + let tr = document.createElement('tr'); + tr.appendChild(this.createCell( + 'no-data', + this.escape(message), + this.countColumns(this.dom.preview) + )); + + this.dom.previewBody.innerHTML = ''; + this.dom.previewBody.appendChild(tr); + } + + // -------------------------------------------------------------------------- + + /** + * Renders a paginator into a container + * + * This mirrors the markup produced by module-admin's + * `admin/views/_components/pagination.php`, which is the source of truth, with + * the same `num_links: 5` / `use_page_numbers: true` configuration. The + * partial cannot be used directly because paging happens over the API with no + * page load, so the whole paginator is redrawn from each response instead. + * + * The links must be anchors: admin styles them as + * `.pagination ul li.page a`, so a button would come out unstyled. + * + * @param {HTMLElement} container The element to render into + * @param {Object} pagination The `meta.pagination` object from the API + * @return {void} + */ + renderPaginator(container, pagination) { + + if (!container) { + return; + } + + let page = pagination.page || 1; + let perPage = pagination.per_page || 0; + let total = pagination.total || 0; + let pages = perPage ? Math.ceil(total / perPage) : 0; + + // "Records X to Y of Z", arithmetic lifted from the partial + let start = (page * perPage) - perPage; + let end = start + perPage; + if (total > 0) { + start++; + } + if (end > total) { + end = total; + } + + let items = []; + + // Like CodeIgniter, render no links at all for an empty or single page + if (pages > 1) { + + if (page > this.numLinks + 1) { + items.push(this.paginatorItem('First', 1, 'first')); + } + + if (page !== 1) { + items.push(this.paginatorItem('‹', page - 1, 'previous')); + } + + let from = Math.max(1, page - this.numLinks); + let to = Math.min(pages, page + this.numLinks); + + for (let i = from; i <= to; i++) { + items.push(i === page + ? `
  • ${i}
    • ` + : this.paginatorItem(String(i), i)); + } + + if (page < pages) { + items.push(this.paginatorItem('›', page + 1, 'next')); + } + + if (page + this.numLinks < pages) { + items.push(this.paginatorItem('Last', pages, 'last')); + } + } + + container.innerHTML = [ + '' + ].join(''); + } + + // -------------------------------------------------------------------------- + + /** + * A single clickable paginator item + * @param {String} label The item's label; may contain entities + * @param {Number} page The page the item navigates to + * @param {String} modifier An extra class for the item, if any + * @return {String} The item's HTML + */ + paginatorItem(label, page, modifier) { + return `
  • ` + + `${label}` + + '
    • '; + } + + // -------------------------------------------------------------------------- + + /** + * Brings the top of the preview back into view + * + * Paging from the bottom paginator replaces the rows above the viewport, so + * without this the page you just asked for is off-screen and has to be + * scrolled to by hand. + * + * @return {void} + */ + scrollToPreview() { + + // Measured rather than hard coded; admin's header is fixed and overlaps + let header = document.querySelector('body > .header'); + let offset = (header ? header.offsetHeight : 0) + 8; + + let top = this.dom.preview.getBoundingClientRect().top + window.scrollY - offset; + + let reduceMotion = window.matchMedia + && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + + window.scrollTo({ + top: Math.max(0, top), + behavior: reduceMotion ? 'auto' : 'smooth' + }); + } + + // -------------------------------------------------------------------------- + + /** + * Creates a table cell + * @param {String|Array} classes The classes to apply + * @param {String} html The cell's contents + * @param {Number} colspan The number of columns to span, if any + * @return {Object} The cell element + */ + createCell(classes, html, colspan) { + + let cell = document.createElement('td'); + + if (Array.isArray(classes)) { + cell.classList.add(...classes); + } else { + cell.classList.add(classes); + } + + if (colspan) { + cell.setAttribute('colspan', colspan); + } + + if (html) { + cell.innerHTML = html; + } + + return cell; + } + + // -------------------------------------------------------------------------- + + /** + * The number of columns in a container's table + * @param {HTMLElement} container The element containing the table + * @return {Number} The column count + */ + countColumns(container) { + return container.querySelectorAll('thead th').length; + } + + // -------------------------------------------------------------------------- + + /** + * Normalises a column label into a cell class, as module-admin does + * @param {String} label The column's label + * @return {String} The cell's classes + */ + fieldClass(label) { + return 'field field--' + String(label) + .toLowerCase() + .replace(/[^a-z0-9 \-_]/g, '') + .replace(/[ _]/g, '-'); + } + + // -------------------------------------------------------------------------- + + /** + * Formats a number as PHP's number_format() does + * + * Deliberately not toLocaleString(); number_format() is not locale aware, so + * this is what actually matches the markup we are mirroring. + * + * @param {Number} value The value to format + * @return {String} The formatted value + */ + numberFormat(value) { + return String(value || 0).replace(/\B(?=(\d{3})+(?!\d))/g, ','); + } + + // -------------------------------------------------------------------------- + + /** + * Returns the shared modal, creating it on first use + * @return {Object|null} The modal instance, or null if admin's Modal is unavailable + */ + getModal() { + + if (this.modal === null) { + try { + this.modal = this.adminController.getInstance('Modal', 'nails/module-admin').create(); + } catch (e) { + this.adminController.warn('Admin\'s Modal component is unavailable', e); + this.modal = false; + } + } + + return this.modal || null; + } + + // -------------------------------------------------------------------------- + + /** + * Shows a message in a modal + * @param {String} title The modal's title + * @param {String} message The message to show + * @return {void} + */ + alert(title, message) { + + let modal = this.getModal(); + + if (!modal) { + window.alert(`${title}\n\n${message}`); + return; + } + + let p = document.createElement('p'); + p.innerText = message; + + // Clear anything a previous confirm() left on the shared instance + modal.onHide(() => { + }); + + modal + .setTitle(this.escape(title)) + .setBody(p) + .clearActions() + .addAction('Close', ['btn-primary'], (event, instance) => instance.hide()) + .show(); + } + + // -------------------------------------------------------------------------- + + /** + * Asks the user to confirm an action + * @param {String} title The modal's title + * @param {String} message The question to ask + * @param {String} action The confirming action's label + * @param {Boolean} warn Whether to present the question as a warning + * @return {Promise} Resolves with whether the action was confirmed + */ + confirm(title, message, action, warn) { + + return new Promise((resolve) => { + + let modal = this.getModal(); + + if (!modal) { + // Better an ugly confirmation than a button which silently does nothing + resolve(window.confirm(message)); + return; + } + + let settled = false; + let settle = (confirmed) => { + if (settled) { + return; + } + settled = true; + resolve(confirmed); + }; + + let p = document.createElement('p'); + p.innerText = message; + + let body = p; + + if (warn) { + + /** + * Wrapped in an alert so the question reads as a warning at a + * glance, rather than as the same paragraph every other + * confirmation shows. Following Notes::showError() in + * module-admin, which hands setBody() an `.alert` of its own - + * the message still goes in through innerText, the alert is the + * only thing being added. + */ + body = document.createElement('div'); + body.classList.add('alert', 'alert-warning'); + + /** + * Inline rather than a rule in this module's stylesheet: the + * admin bundle is global, so `.modal__body > .alert` would + * reach every other admin modal too - module-admin's own alerts + * included - to spare this one the 20px it does not need. + */ + body.style.marginBottom = '0'; + + body.appendChild(p); + } + + // Escape to close is built into the modal, and counts as a cancellation + modal.onHide(() => settle(false)); + + modal + .setTitle(this.escape(title)) + .setBody(body) + .clearActions() + .addAction(this.escape(action), ['btn-primary'], (event, instance) => { + settle(true); + instance.hide(); + }) + .addAction('Cancel', ['btn-danger'], (event, instance) => { + settle(false); + instance.hide(); + }) + .show(); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Routes the submission of flagged forms through a modal confirmation + * + * The listener is on the form rather than the button so that the Delete + * button, which is adopted into the floating controls by the `form` + * attribute, keeps working. + * + * @return {void} + */ + bindConfirmForms() { + + document + .querySelectorAll('form.js-user-import-confirm') + .forEach((form) => { + form.addEventListener('submit', (event) => { + + event.preventDefault(); + + /** + * Held for the whole chain, not just one modal: the shared + * instance has a single onHide slot, so a Delete + * confirmation opening over this one would leave this + * promise unsettled. + */ + this.confirming = true; + + this + .confirmForm(form) + .then((confirmed) => { + if (confirmed) { + /** + * Called off the prototype in case a control is + * ever named "submit", which would shadow it. + * Note this does not fire the submit event, so it + * cannot loop back into this handler. + */ + HTMLFormElement.prototype.submit.call(form); + } + }) + .finally(() => { + this.confirming = false; + }); + }); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Confirms a form's submission, warning about it first if it needs it + * + * A form which names a `data-warn-field` checkbox, and has it unticked, + * gets a warning before the confirmation; Continue is an implied tick, and + * chains straight on into that confirmation. Anything else - no such field, + * or a box already ticked - goes straight there, exactly as every other + * flagged form does. + * + * The two run in sequence over the one shared modal, which keeps them clear + * of module-admin's stacking, and there is nothing to see in between: the + * hide and the re-show both land in the same task. + * + * @param {HTMLFormElement} form The form being submitted + * @return {Promise} Resolves with whether the submission was confirmed + */ + confirmForm(form) { + + let checkbox = this.getWarnCheckbox(form); + + // A local, so the regular confirmation is written once for both routes + let confirmSubmit = () => this.confirm( + form.dataset.confirmTitle || 'Are you sure?', + form.dataset.confirmBody || 'Please confirm you\'d like to continue with this action.', + form.dataset.confirmAction || 'OK' + ); + + if (!checkbox || checkbox.checked) { + return confirmSubmit(); + } + + return this + .confirm( + form.dataset.warnTitle || 'Are you sure?', + form.dataset.warnBody || 'Please confirm you\'d like to continue with this action.', + form.dataset.warnAction || 'Continue', + true + ) + .then((acknowledged) => { + + if (!acknowledged) { + return false; + } + + /** + * The real checkbox, rather than a hidden input, so the value + * reaches the server exactly as a tick would and the page shows + * what was agreed to. It stays ticked if the confirmation is + * then cancelled - the decision was made, and a second Import + * click should not have to make it again. + */ + checkbox.checked = true; + + return confirmSubmit(); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Resolves the checkbox a form wants warning about, if it names one + * + * Looked up through `form.elements` rather than interpolated into a + * selector, so a server-authored name never becomes part of one. + * + * @param {HTMLFormElement} form The form being submitted + * @return {HTMLElement|null} The checkbox, or null if the form has no such field + */ + getWarnCheckbox(form) { + + let name = form.dataset.warnField; + + if (!name) { + return null; + } + + let field = form.elements[name]; + + // A repeated name comes back as a RadioNodeList, which has no `type` + return field && field.type === 'checkbox' ? field : null; + } + + // -------------------------------------------------------------------------- + + /** + * Wires up the Details buttons on both the list and the preview + * + * Delegated from the document for the same two reasons bindDelete() is; see + * the note there. + * + * @return {void} + */ + bindDetails() { + + document.addEventListener('click', (event) => { + + let button = event.target.closest + ? event.target.closest('button.js-user-import-details') + : null; + + if (button) { + event.preventDefault(); + this.showDetails(parseInt(button.dataset.importId, 10)); + } + }); + } + + // -------------------------------------------------------------------------- + + /** + * Shows what went wrong with an import + * + * The modal opens straight away with a spinner rather than after the + * requests land, so a click always does something visible; the body is + * replaced in place once the detail arrives. + * + * @param {Number} id The import's ID + * @return {void} + */ + showDetails(id) { + + let modal = this.getModal(); + let cached = this.imports.find((item) => item.id === id); + + if (!modal) { + window.alert(cached && cached.error + ? cached.error + : 'The details for this import are unavailable.'); + return; + } + + let loading = document.createElement('p'); + let spinner = document.createElement('span'); + spinner.className = 'user-import-spinner'; + spinner.setAttribute('aria-hidden', 'true'); + loading.appendChild(spinner); + loading.appendChild(document.createTextNode(' Loading the details\u2026')); + + // Clear anything a previous confirm() left on the shared instance + modal.onHide(() => { + }); + + modal + .setTitle(`Import #${id}`) + .setBody(loading) + .clearActions() + .addAction('Close', ['btn-primary'], (event, instance) => instance.hide()) + .show(); + + Promise + .all([ + this.request(`${this.endpoint}/${id}`), + this.request(`${this.endpoint}/${id}/items?status=ERROR,WARNING`) + ]) + .then(([job, items]) => { + + let item = job.data || {}; + + modal + .setBody(this.buildDetailsBody(item, items.data || [], items.meta || {})) + .clearActions(); + + if (item.log && item.log.url) { + modal.addAction('Download the full log', ['btn-default'], () => { + window.location = item.log.url; + }); + } + + modal.addAction('Close', ['btn-primary'], (event, instance) => instance.hide()); + }) + .catch((error) => { + + this.adminController.warn('Failed to load user import details', error); + + let p = document.createElement('p'); + p.innerText = error.message || 'The details for this import could not be loaded.'; + + modal.setBody(p); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Builds the details modal's body + * + * Everything goes in via textContent rather than innerHTML. Both blocks + * carry values lifted straight out of the uploaded CSV, so there is no + * escaping decision to get wrong here. + * + * @param {Object} item The import, as returned by the API + * @param {Array} rows The import's failing and warned rows + * @param {Object} meta The rows response's meta + * @return {DocumentFragment} The body + */ + buildDetailsBody(item, rows, meta) { + + let fragment = document.createDocumentFragment(); + + if (item.error) { + let log = document.createElement('pre'); + log.className = 'user-import-details__log'; + log.textContent = item.error; + fragment.appendChild(log); + } + + if (rows.length) { + + let total = (meta.pagination || {}).total || rows.length; + + if (total > rows.length) { + let note = document.createElement('p'); + note.className = 'text-muted'; + note.textContent = 'Showing the first ' + this.numberFormat(rows.length) + + ' of ' + this.numberFormat(total) + ' affected rows; the log has every one.'; + fragment.appendChild(note); + } + + /** + * Each line carries its status, because the two the request asks for + * do not mean the same thing: an ERROR row has no account, a WARNING + * row has one which needs finishing by hand. + */ + let list = document.createElement('pre'); + list.className = 'user-import-details__rows'; + list.textContent = rows + .map((row) => `Line ${row.line} [${row.status}]: ` + + `${row.message || '(no reason was recorded)'}`) + .join('\n'); + fragment.appendChild(list); + } + + /** + * A bad header or a missing CSV fails before a single row is recorded, + * so an empty modal is a real outcome and needs saying out loud. + */ + if (!fragment.childNodes.length) { + let p = document.createElement('p'); + p.textContent = 'No further detail was recorded for this import.'; + fragment.appendChild(p); + } + + return fragment; + } + + // -------------------------------------------------------------------------- + + /** + * Wires up the Delete buttons on both the list and the preview + * + * Delegated from the document rather than from a container, for two + * reasons: the list's buttons are re-rendered on every poll, and the + * preview's button is rendered by admin's floating controls, which sit + * outside `#user-import-preview` - so a listener scoped the way + * bindPreview() scopes its own would never see it. The class is specific + * enough that this costs nothing on the admin pages which have neither. + * + * @return {void} + */ + bindDelete() { + + document.addEventListener('click', (event) => { + + let button = event.target.closest + ? event.target.closest('button.js-user-import-delete') + : null; + + if (button) { + event.preventDefault(); + this.deleteImport(button); + } + }); + } + + // -------------------------------------------------------------------------- + + /** + * Deletes an import, once the user has confirmed it + * + * @param {HTMLElement} button The Delete button which was clicked + * @return {void} + */ + deleteImport(button) { + + let id = parseInt(button.dataset.importId, 10); + + // A confirmation is already up, or this one is already on its way out + if (this.confirming || this.deleting.has(id)) { + return; + } + + /** + * The list knows the item; the preview page does not, and hands the two + * things the copy turns on over as attributes instead. + */ + let item = this.imports.find((x) => x.id === id); + let status = item ? item.status : button.dataset.status; + + // A warned row has an account too, so it is one of the accounts which stays + let created = item + ? (((item.progress || {}).success_count || 0) + ((item.progress || {}).warning_count || 0)) + : parseInt(button.dataset.createdCount || '0', 10); + + let label = button.innerHTML; + + this.confirming = true; + + this + .confirm( + 'Delete this import?', + this.getDeleteConfirmBody(status, created), + 'Delete' + ) + .then((confirmed) => { + + this.confirming = false; + + if (!confirmed) { + return; + } + + /** + * The clicked button is styled here for immediate feedback, and + * the id is recorded so that getActionsCellHtml() reproduces + * that state if the poll re-renders the cell mid-request. + */ + this.deleting.add(id); + button.disabled = true; + button.innerHTML = ''; + + return this + .request(button.dataset.deleteUrl, {method: 'DELETE'}) + .then(() => this.onDeleted(id, button)) + .catch((error) => this.onDeleteFailed(id, error)) + .finally(() => { + + /** + * In `finally`, never in `then`: an id left in the set + * by a failed request would disable that row's button + * on every subsequent poll, with no way back but a + * reload. + */ + this.deleting.delete(id); + + // Only still here if the delete failed; success took the row, or the page + if (button.isConnected) { + button.disabled = false; + button.innerHTML = label; + } + }); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Drops a deleted import, and resyncs + * + * The row goes now rather than on the next poll; at pollIdle that would be + * half a minute of a row which is already gone. loadList() then reconciles + * against the server, and clears the pending timer as it goes. + * + * @param {Number} id The import which was deleted + * @param {HTMLElement} button The button which asked + * @return {void} + */ + onDeleted(id, button) { + + this.removeImport(id); + + // The preview is a page about something which no longer exists + if (button.dataset.redirect) { + window.location = button.dataset.redirect; + return; + } + + this.loadList(); + } + + // -------------------------------------------------------------------------- + + /** + * Reports a delete which did not happen + * + * loadList()'s refusal to raise a modal does not apply here: that is about + * not stacking dialogs every few seconds, and this is a one-off the user + * asked for. + * + * @param {Number} id The import which was not deleted + * @param {Error} error The rejection, carrying .status where we got that far + * @return {void} + */ + onDeleteFailed(id, error) { + + this.adminController.warn('Failed to delete user import', error); + + /** + * Already gone - most likely deleted in another tab. There is nothing + * to apologise for; the row disappearing is the whole message. + */ + if (error.status === 404) { + this.removeImport(id); + this.loadList(); + return; + } + + this.alert( + 'The import could not be deleted', + error.status === 409 + ? 'This import has started running since the page was loaded, so it can no longer be deleted.' + : error.message + ); + + // Whatever the server thinks, the list should agree with it + this.loadList(); + } + + // -------------------------------------------------------------------------- + + /** + * The body of the delete confirmation + * + * What matters is not the status but whether accounts exist: deleting the + * job never deletes the users it created, and a FAILED job may well have + * created some before it stopped - `user_import_item.user_id` is + * ON DELETE SET NULL precisely so that it cannot reach them. Saying + * otherwise invites an admin to read "Delete" as "undo". + * + * @param {String} status The import's status + * @param {Number} created How many accounts it created + * @return {String} The message + */ + getDeleteConfirmBody(status, created) { + + if (created) { + return 'The uploaded CSV and its log will be deleted. The ' + + `${this.numberFormat(created)} user ` + + `${created === 1 ? 'account' : 'accounts'} this import created ` + + 'will NOT be deleted. This cannot be undone.'; + } + + if (status === 'DRAFT' || status === 'PENDING') { + return 'The uploaded CSV will be deleted and no users will be created. This cannot be undone.'; + } + + return 'The uploaded CSV and its log will be deleted. ' + + 'This import created no user accounts. This cannot be undone.'; + } + + // -------------------------------------------------------------------------- + + /** + * Escapes a value for insertion into the DOM + * @param {String} value The value to escape + * @return {String} The escaped value + */ + escape(value) { + let div = document.createElement('div'); + div.appendChild(document.createTextNode(value)); + return div.innerHTML; + } +} + +export default UserImport; diff --git a/assets/sass/admin.scss b/assets/sass/admin.scss index 621a5bc3..1e2b4d63 100644 --- a/assets/sass/admin.scss +++ b/assets/sass/admin.scss @@ -278,3 +278,147 @@ } } } + +/** + * Contextual badges. module-admin ships a bare `.badge` and deliberately no + * variants - it does not import Bootstrap's `_labels.scss`, and `.label` there + * is a form-label selector (`float: left; width: 130px`), so it must not be + * pressed into service as a pill. + * + * Even the neutral case needs an explicit colour: admin's own badge background + * is `rgba(0, 0, 0, 0.25)`, which over a white cell resolves to #bfbfbf - 1.8:1 + * against the white text Bootstrap gives it, a flat contrast failure. + * + * The hexes are admin's `$color-*-text` palette written out longhand, because + * this stylesheet is standalone and cannot reach its variables. All five clear + * 4.5:1 against white at the 12px bold Bootstrap's base rule sets, which also + * supplies the colour, weight and radius. Doubled up on `.badge` so the rules + * do not depend on which stylesheet loads last. + */ +.badge { + &.badge--default { + background-color: #555555; // 7.5:1 + } + + &.badge--info { + background-color: #31708f; // 5.5:1 + } + + &.badge--success { + background-color: #3c763d; // 5.5:1 + } + + &.badge--warning { + background-color: #8a6d3b; // 4.9:1 + } + + &.badge--danger { + background-color: #a94442; // 5.9:1 + } +} + +/** + * A spinner of our own. The admin theme has no reusable one - its only spinner + * is scoped to `.search .mask` - and Bootstrap's `.spinner-border` is not in the + * installed build, so this is self contained. + */ +@keyframes user-import-spin { + to { + transform: rotate(360deg); + } +} + +.user-import-spinner { + display: inline-block; + width: 0.85em; + height: 0.85em; + vertical-align: -0.1em; + border: 2px solid currentColor; + border-right-color: transparent; + border-radius: 50%; + animation: user-import-spin 0.75s linear infinite; +} + +/** + * The details modal's body. Top level, alongside the spinner, because admin's + * Modal appends its container to `document.body` - outside `.module-auth` - so + * a rule nested in there would never match. + */ +.user-import-details { + p { + margin: 0 0 0.5rem; + } + + /** + * The modal body already scrolls (`max-height: 80vh` in module-admin's + * modal.scss), so each block is capped and owns its own scroll; a scroller + * nested inside a scroller makes both awkward to use. + * + * `pre-wrap` is what keeps a long "already registered" message wrapping + * instead of forcing the whole modal sideways. + */ + &__log, + &__rows { + max-height: 40vh; + overflow: auto; + margin: 0 0 0.75rem; + padding: 0.75rem; + font-size: 12px; + line-height: 1.5; + white-space: pre-wrap; + word-break: break-word; + background-color: #f5f5f5; + border: 1px solid #d9d9d9; + border-radius: 4px; + } +} + +.module-auth { + &.import { + /** + * Four btn-xs now that Details is there alongside Review, Log and + * Delete; the 225px this used to carry wrapped them. + */ + th.actions, + td.actions { + width: 275px; + min-width: 275px; + } + } + + &.import--preview { + #user-import-preview { + /** + * Fade the outgoing rows while the next page is in flight. module-admin + * has no busy convention to borrow - the `.search > .mask` spinner's + * `search-spin` keyframes are never declared, so it does not animate - + * so this is the one rule we own. `aria-busy` doubles as the hook, so + * no class has to be invented. + */ + tbody { + transition: opacity 150ms ease-in-out; + } + + &[aria-busy='true'] { + tbody { + opacity: 0.4; + } + + // Anchors have no disabled state, so stop the clicks here too + .pagination a { + pointer-events: none; + } + } + } + + /** + * The line number is a gutter, not data; without a width it takes an equal + * share of the table alongside the CSV's own columns. + */ + th.field--line, + td.field--line { + width: 60px; + text-align: center; + } + } +} diff --git a/auth/config/email_types.php b/auth/config/email_types.php index 4a4d41a4..70acd241 100644 --- a/auth/config/email_types.php +++ b/auth/config/email_types.php @@ -46,6 +46,28 @@ 'can_unsubscribe' => false, 'factory' => Constants::MODULE_SLUG . '::EmailPasswordUpdated', ], + (object) [ + 'slug' => 'user_import_complete', + 'name' => 'Auth: User Import Complete', + 'description' => 'Email sent to the administrator who requested a user import when it finishes.', + 'template_header' => '', + 'template_body' => 'auth/email/user_import_complete', + 'template_footer' => '', + 'default_subject' => 'Your user import has finished', + 'can_unsubscribe' => false, + 'factory' => Constants::MODULE_SLUG . '::EmailUserImportComplete', + ], + (object) [ + 'slug' => 'user_import_failed', + 'name' => 'Auth: User Import Failed', + 'description' => 'Email sent to the administrator who requested a user import when it is rejected.', + 'template_header' => '', + 'template_body' => 'auth/email/user_import_failed', + 'template_footer' => '', + 'default_subject' => 'Your user import was rejected', + 'can_unsubscribe' => false, + 'factory' => Constants::MODULE_SLUG . '::EmailUserImportFailed', + ], (object) [ 'slug' => 'verify_email', 'name' => 'Auth: Verify Email (Generic)', diff --git a/auth/views/email/user_import_complete.php b/auth/views/email/user_import_complete.php new file mode 100644 index 00000000..d4652ea3 --- /dev/null +++ b/auth/views/email/user_import_complete.php @@ -0,0 +1,49 @@ +

    + The user import you requested has finished. +

    + + + + + + + + + + + + + + + + + + + +
    Rows in file{{import.row_count}}
    Accounts created{{import.success}}
    Rows with warnings{{import.warnings}}
    Rows with errors{{import.errors}}
    +{{#error}} +
    {{error}}
    +{{/error}} +{{#import.errors}} +

    + Some rows could not be imported. The log records what happened to every row + in the file, so you can correct the failures and import them again. +

    +{{/import.errors}} +{{#import.warnings}} +

    + Some accounts were created, but something which had to happen afterwards did + not. They exist and can be signed in to; the log identifies them, and + whatever was meant to follow needs finishing by hand. +

    +{{/import.warnings}} +{{#log_url}} +

    + Download the log +

    +{{/log_url}} +{{#import.url}} +

    + View this import in admin +

    +{{/import.url}} diff --git a/auth/views/email/user_import_complete_plaintext.php b/auth/views/email/user_import_complete_plaintext.php new file mode 100644 index 00000000..77f68de3 --- /dev/null +++ b/auth/views/email/user_import_complete_plaintext.php @@ -0,0 +1,29 @@ +The user import you requested has finished. + +Rows in file: {{import.row_count}} +Accounts created: {{import.success}} +Rows with warnings: {{import.warnings}} +Rows with errors: {{import.errors}} +{{#error}} +{{error}} +{{/error}} + +{{#import.errors}} +Some rows could not be imported. The log records what happened to every row in the file, so you can correct the failures and import them again. +{{/import.errors}} + +{{#import.warnings}} +Some accounts were created, but something which had to happen afterwards did not. They exist and can be signed in to; the log identifies them, and whatever was meant to follow needs finishing by hand. +{{/import.warnings}} + +{{#log_url}} +Download the log (you will need to be signed in to admin): + +{{log_url}} +{{/log_url}} +{{#import.url}} + +View this import in admin: + +{{import.url}} +{{/import.url}} diff --git a/auth/views/email/user_import_failed.php b/auth/views/email/user_import_failed.php new file mode 100644 index 00000000..f6da73d9 --- /dev/null +++ b/auth/views/email/user_import_failed.php @@ -0,0 +1,29 @@ +

    + The user import you requested was rejected; no user accounts were created. +

    +{{#error}} +
    {{error}}
    +{{/error}} +{{#import.errors}} +

    + The log records every row which could not be imported, and why. Correct them + in your CSV and upload it again. +

    +{{/import.errors}} +{{#import.warnings}} +

    + Some accounts were created before the import stopped, but something which had + to happen afterwards did not. They exist; the log identifies them, and + whatever was meant to follow needs finishing by hand. +

    +{{/import.warnings}} +{{#log_url}} +

    + Download the log +

    +{{/log_url}} +{{#import.url}} +

    + View this import in admin +

    +{{/import.url}} diff --git a/auth/views/email/user_import_failed_plaintext.php b/auth/views/email/user_import_failed_plaintext.php new file mode 100644 index 00000000..eebe63fe --- /dev/null +++ b/auth/views/email/user_import_failed_plaintext.php @@ -0,0 +1,23 @@ +The user import you requested was rejected; no user accounts were created. + +{{error}} +{{#import.errors}} + +The log records every row which could not be imported, and why. Correct them in your CSV and upload it again. +{{/import.errors}} +{{#import.warnings}} + +Some accounts were created before the import stopped, but something which had to happen afterwards did not. They exist; the log identifies them, and whatever was meant to follow needs finishing by hand. +{{/import.warnings}} +{{#log_url}} + +Download the log (you will need to be signed in to admin): + +{{log_url}} +{{/log_url}} +{{#import.url}} + +View this import in admin: + +{{import.url}} +{{/import.url}} diff --git a/composer.json b/composer.json index 1ed8f7f2..0f19ac59 100644 --- a/composer.json +++ b/composer.json @@ -32,8 +32,10 @@ "php": ">=8.3", "nails/common": "dev-develop", "nails/module-admin": "dev-develop", + "nails/module-api": "dev-develop", "nails/module-captcha": "dev-develop", "nails/module-console": "dev-develop", + "nails/module-cron": "dev-develop", "nails/module-email": "dev-develop", "nails/module-form-builder": "dev-develop", "hybridauth/hybridauth": "~3.0", @@ -44,7 +46,11 @@ }, "require-dev": { "phpunit/phpunit": "^12.0", - "phpstan/phpstan": "^2.0" + "phpstan/phpstan": "^2.0", + "nails/module-queue": "dev-develop" + }, + "suggest": { + "nails/module-queue": "Processes user imports on a long-running worker, rather than in chunks on the cron." }, "scripts": { "test": "./vendor/bin/phpunit", @@ -55,6 +61,11 @@ "Nails\\Auth\\": "src/" } }, + "autoload-dev": { + "psr-4": { + "Tests\\Auth\\": "tests/" + } + }, "extra": { "nails": { "moduleName": "auth", diff --git a/services/services.php b/services/services.php index 99ef2c95..e313bb46 100644 --- a/services/services.php +++ b/services/services.php @@ -42,6 +42,34 @@ return new Service\User\Import(); } }, + 'UserImportCsv' => function (): Service\User\Import\Csv { + if (class_exists('\App\Auth\Service\User\Import\Csv')) { + return new \App\Auth\Service\User\Import\Csv(); + } else { + return new Service\User\Import\Csv(); + } + }, + 'UserImportDispatcher' => function (): Service\User\Import\Dispatcher { + if (class_exists('\App\Auth\Service\User\Import\Dispatcher')) { + return new \App\Auth\Service\User\Import\Dispatcher(); + } else { + return new Service\User\Import\Dispatcher(); + } + }, + 'UserImportProcessor' => function (): Service\User\Import\Processor { + if (class_exists('\App\Auth\Service\User\Import\Processor')) { + return new \App\Auth\Service\User\Import\Processor(); + } else { + return new Service\User\Import\Processor(); + } + }, + 'UserImportValidator' => function (): Service\User\Import\Validator { + if (class_exists('\App\Auth\Service\User\Import\Validator')) { + return new \App\Auth\Service\User\Import\Validator(); + } else { + return new Service\User\Import\Validator(); + } + }, 'UserMeta' => function (): Service\User\Meta { if (class_exists('\App\Auth\Service\User\Meta')) { return new \App\Auth\Service\User\Meta(); @@ -93,6 +121,20 @@ return new Model\User\Group(); } }, + 'UserImport' => function (): Model\User\Import { + if (class_exists('\App\Auth\Model\User\Import')) { + return new \App\Auth\Model\User\Import(); + } else { + return new Model\User\Import(); + } + }, + 'UserImportItem' => function (): Model\User\Import\Item { + if (class_exists('\App\Auth\Model\User\Import\Item')) { + return new \App\Auth\Model\User\Import\Item(); + } else { + return new Model\User\Import\Item(); + } + }, 'UserPassword' => function (): Model\User\Password { // @todo (Pablo 2025-07-15) - this should be a service if (class_exists('\App\Auth\Model\User\Password')) { @@ -145,6 +187,20 @@ return new Factory\Email\PasswordUpdated(); } }, + 'EmailUserImportComplete' => function (): Factory\Email\User\Import\Complete { + if (class_exists('\App\Auth\Factory\Email\User\Import\Complete')) { + return new \App\Auth\Factory\Email\User\Import\Complete(); + } else { + return new Factory\Email\User\Import\Complete(); + } + }, + 'EmailUserImportFailed' => function (): Factory\Email\User\Import\Failed { + if (class_exists('\App\Auth\Factory\Email\User\Import\Failed')) { + return new \App\Auth\Factory\Email\User\Import\Failed(); + } else { + return new Factory\Email\User\Import\Failed(); + } + }, 'EmailVerifyEmail' => function (): Factory\Email\VerifyEmail { if (class_exists('\App\Auth\Factory\Email\VerifyEmail')) { return new \App\Auth\Factory\Email\VerifyEmail(); @@ -204,6 +260,20 @@ return new Resource\User\Group($resource, $model); } }, + 'UserImport' => function ($resource, $model): Resource\User\Import { + if (class_exists('\App\Auth\Resource\User\Import')) { + return new \App\Auth\Resource\User\Import($resource, $model); + } else { + return new Resource\User\Import($resource, $model); + } + }, + 'UserImportItem' => function ($resource, $model): Resource\User\Import\Item { + if (class_exists('\App\Auth\Resource\User\Import\Item')) { + return new \App\Auth\Resource\User\Import\Item($resource, $model); + } else { + return new Resource\User\Import\Item($resource, $model); + } + }, 'UserPasswordHistory' => function ($resource, $model): Resource\User\Password\History { if (class_exists('\App\Auth\Resource\User\Password\History')) { return new \App\Auth\Resource\User\Password\History($resource, $model); diff --git a/src/Admin/Controller/Import.php b/src/Admin/Controller/Import.php index 002eb085..377f5101 100644 --- a/src/Admin/Controller/Import.php +++ b/src/Admin/Controller/Import.php @@ -1,7 +1,15 @@ 'import-user', - 'is_hidden' => true, - 'allowed_types' => 'csv', - ]; + /** + * The bucket the source CSV and its log are stored in + * + * @var array + */ + const array IMPORT_BUCKET = Processor::IMPORT_BUCKET; + + /** + * The permission required to work with imports + * + * @var string + */ + const PERMISSION = Permission\Users\Create::class; + + /** + * Where the controller lives + * + * @var string + */ + const URL = 'admin/auth/import'; + + /** + * The number of per-line errors reported when an upload is rejected + * + * A wholly malformed file produces one per row; the first few say everything + * the thousandth does. Matches Processor::ERROR_SAMPLE_SIZE. + * + * @var int + */ + const ERROR_SAMPLE_SIZE = Processor::ERROR_SAMPLE_SIZE; + + /** + * How long the expiring CDN URL a download hands out is valid for, in seconds + * + * Shorter than module-admin's ADMIN_DATA_EXPORT_URL_TTL (300, see + * Nails\Admin\Service\DataExport::EXPORT_TTL) because the token only has to + * survive the 302 hop; a visitor who was not signed in re-enters log() after + * logging in and is issued a fresh one. + * + * @var int + */ + const int URL_TTL = 60; // -------------------------------------------------------------------------- @@ -62,63 +114,69 @@ public static function announce(): Nav|array|null * * @return void * @throws FactoryException + * @throws ModelException */ public function index(): void { - if (!userHasPermission(Permission\Users\Create::class)) { - unauthorised(); - } - - // -------------------------------------------------------------------------- + $this->assertPermission(); + $this->assertRunning(); /** @var Input $oInput */ $oInput = Factory::service('Input'); /** @var \Nails\Auth\Service\User\Import $oImportService */ $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - if ($oInput->post()) { - try { + try { + /** + * On GET as well as POST: a template which cannot identify an account + * must not be offered for upload in the first place. + */ + $this->assertTemplate(); - if ($oInput->post('action') === 'preview') { - $this - ->validateUpload() - ->renderPreview( - $this->uploadCsv() - ); - - return; + } catch (TemplateException $e) { + $this->oUserFeedback->error($this->escape($e->getMessage())); + $this->data['bTemplateUnusable'] = true; + } - } elseif ($oInput->post('action') === 'import') { - $this - ->validateObject() - ->processImport(); + if ($oInput->post() && empty($this->data['bTemplateUnusable'])) { + try { - } else { - throw new \Exception('Unrecognised action'); - } + $this->handleUpload(); + return; } catch (ValidationException $e) { - $this->oUserFeedback->error( - sprintf( - '%s:
    %s
    ', - $e->getMessage(), + + /** + * Escaped on the way in: UserFeedback messages are rendered raw + * so that the markup below survives, and both the message and + * the itemised errors quote cell and header values lifted + * straight out of the uploaded CSV. + */ + $sMessage = $this->escape($e->getMessage()); + $aErrors = $e->getData() ?? []; + + if (!empty($aErrors)) { + $sMessage .= sprintf( + ':
    %s
    ', implode(';', [ 'max-height: 10rem', 'overflow: auto', 'margin-bottom: 0;', ]), - implode('
    ', $e->getData() ?? []) - ) - ); + implode('
    ', array_map([$this, 'escape'], $aErrors)) + ); + } + + $this->oUserFeedback->error($sMessage); } catch (\Exception $e) { - $this->oUserFeedback->error($e->getMessage()); + $this->oUserFeedback->error($this->escape($e->getMessage())); } } // -------------------------------------------------------------------------- - $this->data['page']->title = 'Import Users'; + $this->data['page']->title = 'Users › Import'; $this->data['additionalFields'] = $oImportService->getAdditionalFields(); Helper::loadView('index'); } @@ -133,6 +191,9 @@ public function index(): void */ public function template(): void { + $this->assertPermission(); + $this->assertTemplate(); + /** @var \Nails\Auth\Service\User\Import $oImportService */ $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); @@ -165,269 +226,349 @@ public function template(): void // -------------------------------------------------------------------------- /** - * Validates the CSV file upload + * Renders the shell of the preview; the rows themselves are paged in from + * the API * - * @return $this + * @return void * @throws FactoryException * @throws ModelException - * @throws NailsException - * @throws ValidationException */ - protected function validateUpload(): self + public function preview(): void { - /** @var Input $oInput */ - $oInput = Factory::service('Input'); - /** @var Cdn\Service\Cdn $oCdn */ - $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + $this->assertPermission(); + $this->assertRunning(); - $aFile = $oInput::file('csv'); - if (empty($aFile)) { - throw new ValidationException('No file selected for upload'); - } + $oImport = $this->getImport(); - if ($aFile['error'] !== UPLOAD_ERR_OK) { - throw new ValidationException( - $oCdn::getUploadError($aFile['error']) - ); - } + /** @var \Nails\Auth\Service\User\Import $oImportService */ + $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - $sMime = $oCdn->getMimeFromFile($aFile['tmp_name']); - if ($aFile['type'] !== 'text/csv' && $sMime !== 'text/plain') { - throw new ValidationException( - 'Uploaded file is not a CSV' - ); + try { + $aKeys = $this->getHeader($oImport); + } catch (ValidationException $e) { + // The CSV has gone; show the columns we would have expected + $this->oUserFeedback->error($this->escape($e->getMessage())); + $aKeys = []; } - $this->validateData( - $this->parseCsv($aFile['tmp_name']) + $this->data['oImport'] = $oImport; + $this->data['bCsvMissing'] = empty($aKeys); + $this->data['aRegistered'] = $this->getRegistered($oImport); + $this->data['aKeys'] = $aKeys ?: $oImportService->getKeys(); + $this->data['sApproveUrl'] = static::URL . '/approve/' . $oImport->id; + $this->data['sListUrl'] = static::URL; + $this->data['page']->title = sprintf( + 'Users › Import › Preview (#%s — %s rows)', + $oImport->id, + $oImport->row_count ?? 0 ); - return $this; + Helper::loadView('preview'); } // -------------------------------------------------------------------------- /** - * @return $this + * Approves a draft import and hands it to a runner + * + * @return void * @throws FactoryException * @throws ModelException - * @throws NailsException - * @throws ValidationException */ - protected function validateObject(): self + public function approve(): void { + $this->assertPermission(); + $this->assertPost(); + /** @var Input $oInput */ $oInput = Factory::service('Input'); - /** @var Cdn\Service\Cdn $oCdn */ - $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); - /** @var Cdn\Model\CdnObject $oObjectModel */ - $oObjectModel = Factory::model('Object', Cdn\Constants::MODULE_SLUG); + /** @var Dispatcher $oDispatcher */ + $oDispatcher = Factory::service('UserImportDispatcher', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); - /** @var Cdn\Resource\CdnObject $oObject */ - $oObject = $oObjectModel->getById((int) $oInput->post('object_id')); - if (empty($oObject)) { - throw new RuntimeException( - 'CDN Object does not exist' - ); - } elseif ($oObject->file->mime !== 'text/csv') { - throw new RuntimeException( - 'Object is not a CSV' - ); - } + $oImport = $this->getImport(); - $sPath = $oCdn->objectLocalPath($oObject->id); - if (empty($sPath)) { - throw new RuntimeException( - 'Failed to get a local path for CSV file.' - ); - } + try { - $this->validateData( - $this->parseCsv($sPath) - ); + if ($oImport->status !== Status::DRAFT) { + throw new ValidationException( + 'Only draft imports can be approved' + ); + } + + $this->assertTemplate(); + + // The file has been sat in the CDN since it was uploaded; make sure + // it is still there, and still makes sense, before committing to it + $oValidator->validateHeader($this->getHeader($oImport)); + + /** + * Registration is state, so it is re-checked here rather than trusted + * from the upload. Without the admin's consent to skip them, a job + * with registered rows would reach a runner only to be rejected, so + * it is refused now while there is somebody to tell. + */ + $bSkipRegistered = (bool) $oInput->post('skip_registered'); + $aRegistered = $this->getRegistered($oImport); + + if (!empty($aRegistered) && !$bSkipRegistered) { + $this->assertNoErrors($this->flattenRowErrors($aRegistered)); + } + + $oModel->update($oImport->id, [ + 'skip_registered' => $bSkipRegistered, + ]); + + $oDispatcher->dispatch($oImport); - return $this; + $this->oUserFeedback->success(sprintf( + 'Import #%s has been queued and will begin shortly.', + $oImport->id + )); + + } catch (\Exception $e) { + $this->oUserFeedback->error($this->escape($e->getMessage())); + } + + redirect(static::URL); } // -------------------------------------------------------------------------- /** - * Validates the CSV data + * Redirects to a short-lived URL for the job's source CSV * - * @param array $aData The data to validate - * - * @return $this + * @return void * @throws FactoryException - * @throws ValidationException * @throws ModelException - * @throws NailsException */ - protected function validateData(array $aData): self + public function source(): void { - /** @var \Nails\Auth\Service\User\Import $oImportService */ - $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - /** @var FormValidation $oFormValidationService */ - $oFormValidationService = Factory::service('FormValidation'); + $this->assertPermission(); + $this->download($this->getImport()->object_id); + } - $aHeader = array_splice($aData, 0, 1); - $aHeader = reset($aHeader); + // -------------------------------------------------------------------------- - // Validate Header Row - if (empty($aHeader)) { - throw new ValidationException( - 'Missing header row' - ); - } + /** + * Redirects to a short-lived URL for the job's log + * + * @return void + * @throws FactoryException + * @throws ModelException + */ + public function log(): void + { + $this->assertPermission(); + $this->download($this->getImport()->log_id); + } - $aKeys = $oImportService->getKeys(); - $aDiff = array_diff($aHeader, $aKeys); - if (!empty($aDiff)) { - throw new ValidationException(sprintf( - 'Header row contains the following invalid values: %s', - implode(', ', $aDiff) - )); - } + // -------------------------------------------------------------------------- - // Validate data - $aErrors = []; - $iLines = 2; // Skip the header + /** + * Redirects to a short-lived URL for one of the job's CDN objects + * + * The source CSV and the log both contain personal data, so neither is ever + * linked directly: the URL is minted here, behind the permission check, and + * expires. It does end up in the address bar and the Referer of the CDN + * request - the same trade module-admin's data export makes - which is why + * the TTL is measured in seconds. + * + * Reading personal data sits behind a create permission because that is the + * permission which governs the whole of this controller; the ability to run + * an import is the ability to read its file. + * + * @param int|null $iObjectId The CDN object to hand out + * + * @return void + */ + protected function download(?int $iObjectId): void + { + if (empty($iObjectId)) { + // No log was attached, or the object has since been destroyed + show404(); + } - // Key the rules by the header's own columns; the CSV need not use every - // key, nor list them in the same order as getKeys() - $aValidationRules = array_filter( - array_map( - fn($sKey) => $oImportService->getValidationRules($sKey), - array_combine($aHeader, $aHeader) - ) - ); + redirect(cdnExpiringUrl($iObjectId, static::URL_TTL, true)); + } - foreach ($aData as $aDatum) { + // -------------------------------------------------------------------------- - try { + /** + * Validates the upload, stores it, and records the job + * + * Nothing is persisted unless the file is usable; a rejected upload leaves + * neither a CDN object nor a job behind. + * + * @throws FactoryException + * @throws ModelException + * @throws NailsException + * @throws ValidationException + */ + protected function handleUpload(): void + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + + $aFile = $this->validateUpload(); + $iRowCount = $oCsv->countRows($aFile['tmp_name']); + $oObject = $this->uploadCsv(); + + $iId = $oModel->create([ + 'object_id' => $oObject->id, + 'additional' => json_encode((object) ($oInput->post('additional') ?: [])), + 'row_count' => $iRowCount, + 'status' => Status::DRAFT->value, + ]); + + if (empty($iId)) { + throw new RuntimeException(sprintf( + 'Failed to record the import; %s', + $oModel->lastError() + )); + } - $aDatum = array_combine($aHeader, $aDatum); - $aDatum = array_map('trim', $aDatum); + redirect(static::URL . '/preview/' . $iId); + } - // Basic validation - $oFormValidationService - ->buildValidator( - aRules: $aValidationRules, - aData: $aDatum - ) - ->run(); + // -------------------------------------------------------------------------- - } catch (ValidationException $e) { + /** + * Validates the CSV file upload + * + * Only whole-of-file concerns are checked here — that there is a file, that + * it is a CSV, that its header is one we understand, and that it does not + * contradict itself. Row level validation is the runner's job. + * + * @return array The $_FILES entry + * @throws FactoryException + * @throws ModelException + * @throws NailsException + * @throws ValidationException + */ + protected function validateUpload(): array + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); + /** @var Cdn\Service\Cdn $oCdn */ + $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); - foreach ($e->getData() as $key => $error) { - $aErrors[] = sprintf( - 'Line %d: %s: %s', - $iLines, - $key, - $error - ); - } + $aFile = $oInput::file('csv'); + if (empty($aFile)) { + throw new ValidationException('No file selected for upload'); + } - } catch (Throwable $e) { - $aErrors[] = sprintf( - 'Error on line %d: %s', - $iLines, - $e->getMessage() - ); - } + if ($aFile['error'] !== UPLOAD_ERR_OK) { + throw new ValidationException( + $oCdn::getUploadError($aFile['error']) + ); + } - $iLines++; + $sMime = $oCdn->getMimeFromFile($aFile['tmp_name']); + if ($aFile['type'] !== 'text/csv' && $sMime !== 'text/plain') { + throw new ValidationException( + 'Uploaded file is not a CSV' + ); } - // Duplicate detection cannot live in the per-field rules; those only ever - // see a single value, and the validator abandons a field's remaining rules - // as soon as one of them fails. It's a whole-of-file concern, so handle it - // as one pass over the parsed data. - $aErrors = array_merge( - $aErrors, - $this->detectDuplicates($aHeader, $aData) - ); + $aHeader = $oCsv->getHeader($aFile['tmp_name']); - if (!empty($aErrors)) { + $oValidator->validateHeader($aHeader); - $message = count($aErrors) === 1 - ? '1 error was found in the CSV file' - : sprintf('%d errors were found in the CSV file', count($aErrors)); + $this->assertNoErrors( + $oValidator->detectDuplicates($aFile['tmp_name']) + ); - throw (new ValidationException($message)) - ->setData($aErrors); - } + /** + * Row validation used to be left entirely to the runner, which meant an + * admin could approve - and commit to - a file which could never import. + * The rows are streamed, so this holds no more than one at a time, and + * the request already reads the whole file twice before reaching here. + * + * Only "the CSV is wrong" problems are checked. Values which are already + * registered are a separate, skippable concern handled at preview; see + * Import\Validator::detectRegistered(). + */ + $this->assertNoErrors( + $this->flattenRowErrors( + $oValidator->validateRows($aHeader, $oCsv->readRawRows($aFile['tmp_name'])) + ) + ); - return $this; + return $aFile; } // -------------------------------------------------------------------------- /** - * Detects values which are duplicated within the CSV itself + * Renders per-line errors as flat, prefixed strings * - * @param array $aHeader The CSV's header row - * @param array $aData The CSV's data rows, header removed + * Matches detectDuplicates()'s output so both can go through the same + * reporting path. + * + * @param array $aErrors Errors keyed by line number * * @return string[] - * @throws FactoryException */ - protected function detectDuplicates(array $aHeader, array $aData): array + protected function flattenRowErrors(array $aErrors): array { - /** @var \Nails\Auth\Service\User\Import $oImportService */ - $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - - $aErrors = []; + $aOut = []; - foreach ($oImportService->getUniqueKeys() as $sKey) { - - $iColumn = array_search($sKey, $aHeader, true); - if ($iColumn === false) { - continue; - } - - $aSeen = []; - - foreach ($aData as $iIndex => $aDatum) { - - $sValue = strtolower(trim($aDatum[$iColumn] ?? '')); - if ($sValue === '') { - continue; - } - - $iLine = $iIndex + 2; // Lines are 1 indexed, and the header is line 1 - - if (array_key_exists($sValue, $aSeen)) { - $aErrors[] = sprintf( - 'Line %d: %s: "%s" must only appear once; it is also on line %d', - $iLine, - $sKey, - $sValue, - $aSeen[$sValue] - ); - } else { - $aSeen[$sValue] = $iLine; - } + foreach ($aErrors as $iLine => $aLineErrors) { + foreach ($aLineErrors as $sError) { + $aOut[] = sprintf('Line %d: %s', $iLine, $sError); } } - return $aErrors; + return $aOut; } // -------------------------------------------------------------------------- /** - * Parses the CSV file into an array + * Rejects the upload if anything was found * - * @param string $sPath The path to the CSV + * The list is capped: a wholly malformed file can produce an error for every + * row, and thousands of near-identical lines tell an admin less than the + * first few do. * - * @return array + * @param string[] $aErrors + * + * @throws ValidationException */ - protected function parseCsv(string $sPath): array + protected function assertNoErrors(array $aErrors): void { - return array_map( - fn($line) => str_getcsv($line, escape: ''), - file($sPath) - ); + if (empty($aErrors)) { + return; + } + + $iTotal = count($aErrors); + + $sMessage = $iTotal === 1 + ? '1 error was found in the CSV file' + : sprintf('%s errors were found in the CSV file', number_format($iTotal)); + + if ($iTotal > static::ERROR_SAMPLE_SIZE) { + $aErrors = array_slice($aErrors, 0, static::ERROR_SAMPLE_SIZE); + $aErrors[] = sprintf( + '…and %s more', + number_format($iTotal - static::ERROR_SAMPLE_SIZE) + ); + } + + throw (new ValidationException($sMessage)) + ->setData($aErrors); } // -------------------------------------------------------------------------- @@ -447,12 +588,12 @@ protected function uploadCsv(): Cdn\Resource\CdnObject $oObject = $oCdn->objectCreate( 'csv', - self::IMPORT_BUCKET, + static::IMPORT_BUCKET, [ 'Content-Type' => 'text/csv', 'metadata' => [ [ - 'key' => (new SystemKey\UserImport)->get(), + 'key' => (new SystemKey\UserImport())->get(), 'value' => true, ], ], @@ -474,209 +615,178 @@ protected function uploadCsv(): Cdn\Resource\CdnObject // -------------------------------------------------------------------------- /** - * @param Cdn\Resource\CdnObject $oObject + * Returns the import named by the URL, or 404s * - * @return void * @throws FactoryException - * @throws NailsException + * @throws ModelException */ - protected function renderPreview(Cdn\Resource\CdnObject $oObject): void + protected function getImport(): Resource\User\Import { - /** @var \Nails\Auth\Service\User\Import $oImportService */ - $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - /** @var Input $oInput */ - $oInput = Factory::service('Input'); - /** @var Cdn\Service\Cdn $oCdn */ - $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); - - $sPath = $oCdn->objectLocalPath($oObject->id); - if (empty($sPath)) { - throw new RuntimeException( - 'Failed to get a local path for CSV file' - ); - } + /** @var Uri $oUri */ + $oUri = Factory::service('Uri'); + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); - $aData = $this->parseCsv($sPath); + /** @var Resource\User\Import|null $oImport */ + $oImport = $oModel->getById((int) $oUri->segment(5)); - $aKeys = $oImportService->getKeys(); - $aHeader = array_splice($aData, 0, 1); - $aHeader = reset($aHeader); - - foreach ($aData as &$aDatum) { - $aDatum = array_combine($aHeader, $aDatum); - foreach ($aKeys as $sField) { - if ($aDatum[$sField] === '') { - $aDatum[$sField] = $oImportService->getDefaultValue($sField); - } - } + if (empty($oImport)) { + show404(); } - $this->data['aKeys'] = $aKeys; - $this->data['aHeader'] = $aHeader; - $this->data['aData'] = $aData; - $this->data['oObject'] = $oObject; - $this->data['oAdditional'] = (object) $oInput->post('additional'); + return $oImport; + } - $this->data['page']->title = 'Import Users: Preview (' . count($aData) . ')'; + // -------------------------------------------------------------------------- - Helper::loadView('preview'); + /** + * Returns the header row of an import's CSV + * + * @return string[] + * @throws FactoryException + * @throws ValidationException + */ + protected function getHeader(Resource\User\Import $oImport): array + { + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + + return $oCsv->getHeader($this->getCsvPath($oImport)); } // -------------------------------------------------------------------------- /** - * Process the import + * Returns the local path of an import's CSV * - * @return void * @throws FactoryException - * @throws NailsException + * @throws ValidationException If the file is no longer retrievable */ - protected function processImport(): void + protected function getCsvPath(Resource\User\Import $oImport): string { - /** @var \Nails\Auth\Service\User\Import $oImportService */ - $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - /** @var Input $oInput */ - $oInput = Factory::service('Input'); /** @var Cdn\Service\Cdn $oCdn */ $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); - /** @var User $oUserModel */ - $oUserModel = Factory::model('User', Constants::MODULE_SLUG); - $iObjectId = (int) $oInput->post('object_id'); - $oAdditional = json_decode($oInput->post('additional')); - - $sPath = $oCdn->objectLocalPath($iObjectId); + $sPath = $oCdn->objectLocalPath($oImport->object_id); if (empty($sPath)) { - throw new RuntimeException( - 'Failed to get a local path for CSV file.' + throw new ValidationException( + 'Failed to get a local path for the CSV file' ); } - $aKeys = $oImportService->getKeys(); - $aData = $this->parseCsv($sPath); - $aHeader = array_splice($aData, 0, 1); - $aHeader = reset($aHeader); - $iSuccess = 0; - $iError = 0; - $aLog = []; - - foreach ($aData as $aDatum) { - - $aDatum = array_combine($aHeader, $aDatum); - $bSendEmail = stringToBoolean($aDatum['send_email'] ?? false); - - $aUserData = []; - foreach ($aKeys as $sKey) { - $aUserData[$sKey] = $aDatum[$sKey] ?? null; - if ($aUserData[$sKey] === '') { - $aUserData[$sKey] = $oImportService->getDefaultValue($sKey); - } - } + return $sPath; + } - // Apply additional fields - foreach ($oAdditional as $oProperty => $mValue) { - $aUserData[$oProperty] = $oImportService->parseAdditionalFields($oProperty, $mValue); - } + // -------------------------------------------------------------------------- - try { + /** + * @throws FactoryException + */ + /** + * Escapes a value destined for a user feedback message + * + * UserFeedback messages are rendered raw - see module-admin's + * page-header.php - and every message this controller reports quotes header + * or cell values taken from an uploaded CSV, so they are escaped here rather + * than trusted downstream.§ + */ + protected function escape(?string $sValue): string + { + return htmlspecialchars((string) $sValue, ENT_QUOTES, 'UTF-8'); + } - $oUser = $oUserModel->create($aUserData, $bSendEmail); - - if ($oUser) { - $iSuccess++; - $aLog[] = array_merge( - $aDatum, - [ - 'id' => $oUser->id, - 'status' => 'SUCCESS', - 'message' => '', - ] - ); - } else { - $iError++; - $aLog[] = array_merge( - $aDatum, - [ - 'id' => null, - 'status' => 'ERROR', - 'message' => $oUserModel->lastError(), - ] - ); - } + // -------------------------------------------------------------------------- - } catch (Throwable $e) { - $iError++; - $aLog[] = array_merge( - $aDatum, - [ - 'id' => null, - 'status' => 'ERROR', - 'message' => $e->getMessage(), - ] - ); - } + /** + * Returns the job's rows whose unique values are already registered + * + * Recomputed rather than stored: registration is state which can change + * between upload and approval, and a stale answer here would either block a + * job which is now fine or wave through one which is not. + * + * Only meaningful before the job runs, so a job which is past DRAFT is not + * made to pay for the pass. + * + * @return array Errors keyed by line number + * @throws FactoryException + */ + protected function getRegistered(Resource\User\Import $oImport): array + { + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + + if ($oImport->status !== Status::DRAFT) { + return []; } - array_unshift($aLog, array_merge( - $aHeader, - [ - 'id', - 'status', - 'message', - ] - )); + try { + $sPath = $this->getCsvPath($oImport); - // Convert array to CSV and save to cDN - $fp = fopen('php://temp', 'r+'); - foreach ($aLog as $row) { - fputcsv($fp, $row, escape: ''); + } catch (ValidationException $e) { + // The CSV has gone; the caller has already reported that + return []; } - rewind($fp); - $csvLog = stream_get_contents($fp); - fclose($fp); - - /** @var \DateTime $oNow */ - $oNow = Factory::factory('DateTime'); - $oLog = $oCdn->objectCreate( - $csvLog, - self::IMPORT_BUCKET, - [ - 'no-md5-check' => true, - 'Content-Type' => 'text/csv', - 'filename_display' => sprintf( - 'user-import-log-%s.csv', - $oNow->format('Y-m-d_H-i-s') - ), - 'metadata' => [ - [ - 'key' => (new SystemKey\UserImport)->get(), - 'value' => true, - ], - [ - 'key' => (new SystemKey\ImportedFrom)->get(), - 'value' => $iObjectId, - ], - ], - ], - true + + return $oValidator->detectRegistered( + $oCsv->getHeader($sPath), + $oCsv->readRawRows($sPath) ); + } - if (!empty($iSuccess)) { - $this->oUserFeedback->success(sprintf( - '%s user accounts created successfully. See log for details.', - $iSuccess, - cdnServe($oLog->id, true) - )); + // -------------------------------------------------------------------------- + + /** + * Asserts the import template can produce a usable account + * + * @throws TemplateException + * @throws FactoryException + */ + protected function assertTemplate(): void + { + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + protected function assertPermission(): void + { + if (!userHasPermission(static::PERMISSION)) { + unauthorised(); } + } - if (!empty($iError)) { - $this->oUserFeedback->error(sprintf( - '%s user accounts encountered errors. See log for details.', - $iError, - cdnServe($oLog->id, true) - )); + protected function assertRunning(): void + { + /** @var Dispatcher $oDispatcher */ + $oDispatcher = Factory::service('UserImportDispatcher', Constants::MODULE_SLUG); + + if (!$oDispatcher->isRunning()) { + $this->oUserFeedback->warning( + 'The user import cron job is not running' . + '
    The cron job has not been executed within the past 5 minutes; imports will not be processed.' + ); } + } + + // -------------------------------------------------------------------------- + + /** + * State changes are POST only so they cannot be triggered by a link + * + * @throws FactoryException + */ + protected function assertPost(): void + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); - redirect(self::url()); + if (strtoupper((string) $oInput::server('REQUEST_METHOD')) !== 'POST') { + show404(); + } } } diff --git a/src/Api/Controller/Import.php b/src/Api/Controller/Import.php new file mode 100644 index 00000000..90c55b96 --- /dev/null +++ b/src/Api/Controller/Import.php @@ -0,0 +1,440 @@ +status->isDeletable() || $oItem->claim_token !== null) + ) { + throw new ApiException( + 'An import which is in progress cannot be deleted', + $oHttpCodes::STATUS_CONFLICT + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Formats the response object + * + * @param Resource\User\Import $oObj The object to format + * + * @throws FactoryException + */ + protected function formatObject($oObj): stdClass + { + return (object) [ + 'id' => $oObj->id, + 'status' => $oObj->status->value, + 'runner' => $oObj->runner?->value, + 'progress' => (object) [ + 'row_count' => $oObj->row_count, + 'validated_count' => $oObj->validated_count, + 'processed_count' => $oObj->processed_count, + 'success_count' => $oObj->success_count, + 'warning_count' => $oObj->warning_count, + 'error_count' => $oObj->error_count, + 'percent' => $oObj->getPercent(), + ], + /** + * Both URLs are the guarded admin routes rather than CDN objects: + * the files carry personal data, so the URL which reaches them is + * minted behind a permission check and expires. + * + * They sit on `source` and `log` rather than on the `urls` object + * below because `item.log.url` is what the admin JS already reads; + * the inconsistency is deliberate. `source.url` has no consumer + * yet, and is exposed so that a future link cannot be tempted back + * to a direct CDN URL. + */ + 'source' => (object) [ + 'id' => $oObj->object?->id, + 'filename' => $oObj->object?->file?->name?->human, + 'url' => siteUrl('admin/auth/import/source/' . $oObj->id), + ], + 'log' => $oObj->log_id + ? (object) [ + 'id' => $oObj->log_id, + 'url' => siteUrl('admin/auth/import/log/' . $oObj->id), + ] + : null, + 'error' => $oObj->error, + /** + * The stored error is deliberately multi-line - see + * Processor::composeError() - so anywhere with room for a single + * line takes the summary rather than doing its own string surgery. + */ + 'error_summary' => $oObj->error + ? explode("\n", $oObj->error, 2)[0] + : null, + 'created' => $oObj->created, + 'started' => $oObj->started, + 'finished' => $oObj->finished, + 'user' => $oObj->user + ? (object) [ + 'id' => $oObj->user->id, + 'name' => $oObj->user->name, + 'email' => $oObj->user->email, + ] + : null, + 'urls' => (object) [ + 'preview' => siteUrl('admin/auth/import/preview/' . $oObj->id), + 'approve' => siteUrl('admin/auth/import/approve/' . $oObj->id), + 'delete' => siteUrl('api/auth/import/' . $oObj->id), + ], + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes a job, and the CDN objects behind it + * + * @param ApiResponse $oApiResponse The API response + * @param Resource\User\Import $oItem The job being deleted + * + * @throws ApiException + * @throws FactoryException + * @throws ModelException + */ + protected function delete(ApiResponse $oApiResponse, Entity $oItem): void + { + // Applies the guard in userCan(), and 500s if the row will not go + parent::delete($oApiResponse, $oItem); + + /** + * Only now that the row is gone: object_id is ON DELETE CASCADE, so + * destroying the CSV first would take the job with it - inside + * objectDestroy()'s own transaction - and the delete above would then + * report a job which no longer exists. + * + * A failure here is logged rather than raised: the row has gone and the + * delete has genuinely succeeded, so there is nothing for the caller to + * retry. It cannot be left silent though - `auth:user:import:clean` + * reaps by walking job rows, so an object orphaned here is unreachable + * by anything else, and the log line is the only thread back to what it + * was for. + */ + /** @var Model\User\Import $oModel */ + $oModel = $this->oModel; + /** @var Logger $oLogger */ + $oLogger = Factory::service('Logger'); + + foreach ($oModel->destroyObjects($oItem) as $iObjectId => $sError) { + $oLogger->error(sprintf( + 'Failed to destroy CDN object #%s belonging to user import #%s; %s', + $iObjectId, + $oItem->id, + $sError + )); + } + } + + // -------------------------------------------------------------------------- + + /** + * Returns a page of the job's CSV, as it will be imported + * + * Reached at GET /api/auth/import/{id}/rows; CrudController::read() routes + * the trailing segment to a same-named method. + * + * @param ApiResponse $oApiResponse The API response + * @param Resource\User\Import $oItem The job being previewed + * + * @throws ApiException + * @throws FactoryException + * @throws NailsException + */ + protected function rows(ApiResponse $oApiResponse, Entity $oItem): void + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); + /** @var HttpCodes $oHttpCodes */ + $oHttpCodes = Factory::service('HttpCodes'); + /** @var Cdn\Service\Cdn $oCdn */ + $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + + $sPath = $oCdn->objectLocalPath($oItem->object->id); + if (empty($sPath)) { + throw new ApiException( + 'The CSV for this import is no longer available', + $oHttpCodes::STATUS_NOT_FOUND + ); + } + + $iPage = max(1, (int) $oInput->get(static::CONFIG_PAGE_PARAM)); + $iOffset = ($iPage - 1) * static::CONFIG_PER_PAGE; + + /** + * row_count is recorded when the file is uploaded, so paging does not + * have to re-scan the file on every request. + */ + $iTotal = $oItem->row_count ?? $oCsv->countRows($sPath); + + $aRows = []; + foreach ($oCsv->readRows($sPath, $iOffset, static::CONFIG_PER_PAGE) as $iLine => $aRow) { + $aRows[] = (object) [ + 'line' => $iLine, + 'data' => (object) $aRow, + ]; + } + + $oApiResponse + ->setData($aRows) + ->setMeta([ + 'header' => $oCsv->getHeader($sPath), + 'pagination' => [ + 'page' => $iPage, + 'per_page' => static::CONFIG_PER_PAGE, + 'total' => $iTotal, + 'previous' => $this->buildUrl($iTotal, $iPage, -1), + 'next' => $this->buildUrl($iTotal, $iPage, 1), + ], + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Returns a page of the job's per-row outcomes + * + * Reached at GET /api/auth/import/{id}/items; CrudController::read() routes + * the trailing segment to a same-named method, as it does for rows(). + * + * Kept off formatObject() deliberately: that runs for every row of the list, + * and the list polls while any job is active, so folding per-row detail into + * it would mean an extra query per job per poll for something nobody is + * looking at until they click. + * + * @param ApiResponse $oApiResponse The API response + * @param Resource\User\Import $oItem The job being inspected + * + * @throws ApiException + * @throws FactoryException + * @throws ModelException + */ + protected function items(ApiResponse $oApiResponse, Entity $oItem): void + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + $aData = [ + 'where' => [['import_id', $oItem->id]], + 'sort' => [['line', 'ASC']], + ]; + + /** + * A comma-separated list rather than a single value, so the details + * modal can ask for the errors and the warnings - two statuses which + * mean different things to the reader, but which are shown together - in + * one request rather than two. + */ + $aStatuses = $this->readStatuses((string) $oInput->get('status')); + + if (!empty($aStatuses)) { + $aData['where_in'] = [['status', $aStatuses]]; + } + + $iPage = max(1, (int) $oInput->get(static::CONFIG_PAGE_PARAM)); + $iTotal = $oItemModel->countAll($aData); + + $aItems = array_map( + fn(Resource\User\Import\Item $oRow): stdClass => (object) [ + 'line' => $oRow->line, + 'status' => $oRow->status->value, + 'message' => $oRow->message, + 'user_id' => $oRow->user_id, + ], + $oItemModel->getAll($iPage, static::CONFIG_PER_PAGE, $aData) + ); + + $oApiResponse + ->setData($aItems) + ->setMeta([ + 'total_errors' => $oItem->error_count, + 'total_warnings' => $oItem->warning_count, + 'pagination' => [ + 'page' => $iPage, + 'per_page' => static::CONFIG_PER_PAGE, + 'total' => $iTotal, + 'previous' => $this->buildUrl($iTotal, $iPage, -1), + 'next' => $this->buildUrl($iTotal, $iPage, 1), + ], + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Reads the `status` filter + * + * Every value is checked, and an unknown one is refused rather than dropped: + * a filter which quietly ignores what it was asked for would answer a + * question nobody asked. + * + * @param string $sStatus The requested status, or a comma-separated list of them + * + * @return string[] The statuses to filter by, empty for no filter + * @throws ApiException If any of them is not a status + * @throws FactoryException + */ + protected function readStatuses(string $sStatus): array + { + /** @var HttpCodes $oHttpCodes */ + $oHttpCodes = Factory::service('HttpCodes'); + + $aStatuses = []; + + foreach (array_filter(array_map('trim', explode(',', $sStatus))) as $sCandidate) { + + $oStatus = ItemStatus::tryFrom($sCandidate); + + if ($oStatus === null) { + throw new ApiException( + sprintf('"%s" is not a valid item status', $sCandidate), + $oHttpCodes::STATUS_BAD_REQUEST + ); + } + + $aStatuses[] = $oStatus->value; + } + + return array_values(array_unique($aStatuses)); + } +} diff --git a/src/Console/Command/User/Import/Clean.php b/src/Console/Command/User/Import/Clean.php new file mode 100644 index 00000000..4b8d3afc --- /dev/null +++ b/src/Console/Command/User/Import/Clean.php @@ -0,0 +1,261 @@ +setName('auth:user:import:clean') + ->setDescription('Releases orphaned user import jobs and reaps old ones'); + } + + // -------------------------------------------------------------------------- + + /** + * Executes the command + * + * @param InputInterface $oInput The Input Interface provided by Symfony + * @param OutputInterface $oOutput The Output Interface provided by Symfony + */ + protected function execute(InputInterface $oInput, OutputInterface $oOutput): int + { + parent::execute($oInput, $oOutput); + + try { + + $this->banner('User Import: Clean'); + $this + ->releaseOrphans() + ->reapDrafts() + ->rotateFinished(); + + } catch (Throwable $e) { + return $this->abort( + self::EXIT_CODE_FAILURE, + [$e->getMessage()] + ); + } + + $oOutput->writeln(''); + $oOutput->writeln('Complete!'); + + return self::EXIT_CODE_SUCCESS; + } + + // -------------------------------------------------------------------------- + + /** + * Releases claims held by processes which are no longer with us + * + * The status is deliberately left alone; a job resumes from its cursor, and + * sending it back to PENDING would restart it from the top and re-validate + * rows whose users have since been created. + * + * @throws FactoryException + * @throws ModelException + */ + protected function releaseOrphans(): self + { + $this->oOutput->writeln('Releasing orphaned claims'); + + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $oDb + ->set('claim_token', null) + ->set('claimed', null) + ->where('claim_token IS NOT NULL', null, false) + ->where('claimed <', $this->getCutOff('AUTH_USER_IMPORT_STALE_CLAIM', static::STALE_CLAIM)) + ->update($oModel->getTableName()); + + $this->oOutput->writeln(sprintf( + 'Released %s', + $oDb->affected_rows() + )); + + return $this; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes uploads which were never approved + * + * @throws FactoryException + * @throws ModelException + */ + protected function reapDrafts(): self + { + $this->oOutput->writeln(''); + $this->oOutput->writeln('Reaping abandoned drafts'); + + $iDeleted = $this->deleteJobs( + [Status::DRAFT], + $this->getCutOff('AUTH_USER_IMPORT_DRAFT_TTL', static::DRAFT_TTL) + ); + + $this->oOutput->writeln(sprintf('Deleted %s', $iDeleted)); + + return $this; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes jobs which finished long enough ago that nobody is coming back for them + * + * @throws FactoryException + * @throws ModelException + */ + protected function rotateFinished(): self + { + $this->oOutput->writeln(''); + $this->oOutput->writeln('Rotating finished jobs'); + + $iDeleted = $this->deleteJobs( + Status::terminal(), + $this->getCutOff('AUTH_USER_IMPORT_RETENTION', static::RETENTION) + ); + + $this->oOutput->writeln(sprintf('Deleted %s', $iDeleted)); + + return $this; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes jobs of the given statuses which were last modified before the cut off + * + * @param Status[] $aStatuses + * + * @throws FactoryException + * @throws ModelException + */ + protected function deleteJobs(array $aStatuses, string $sCutOff): int + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aRows = $oDb + ->select('id') + ->where_in('status', Status::values($aStatuses)) + ->where('modified <', $sCutOff) + ->order_by('id', 'asc') + ->limit(static::MAX_PER_RUN) + ->get($oModel->getTableName()) + ->result(); + + $iDeleted = 0; + + foreach ($aRows as $oRow) { + + /** @var Resource\User\Import|null $oImport */ + $oImport = $oModel->getById((int) $oRow->id); + if (empty($oImport)) { + continue; + } + + // The job goes first; the CDN objects cascade onto it, and a + // half-deleted job is worse than a lingering file. + if (!$oModel->delete($oImport->id)) { + $this->oOutput->writeln(sprintf( + '↳ Failed to delete import #%s; %s', + $oImport->id, + $oModel->lastError() + )); + continue; + } + + foreach ($oModel->destroyObjects($oImport) as $iObjectId => $sError) { + $this->oOutput->writeln(sprintf( + '↳ Failed to destroy CDN object #%s; %s', + $iObjectId, + $sError + )); + } + + $iDeleted++; + } + + return $iDeleted; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the datetime $sConfigKey seconds ago + * + * @throws FactoryException + */ + protected function getCutOff(string $sConfigKey, int $iDefault): string + { + $iSeconds = (int) Config::get($sConfigKey, $iDefault) ?: $iDefault; + + /** @var \DateTime $oCutOff */ + $oCutOff = Factory::factory('DateTime'); + $oCutOff->sub(new DateInterval('PT' . $iSeconds . 'S')); + + return $oCutOff->format('Y-m-d H:i:s'); + } +} diff --git a/src/Console/Command/User/Import/Process.php b/src/Console/Command/User/Import/Process.php new file mode 100644 index 00000000..2e43d34d --- /dev/null +++ b/src/Console/Command/User/Import/Process.php @@ -0,0 +1,312 @@ +setName('auth:user:import:process') + ->setDescription('Processes any approved user import jobs') + ->addOption( + 'chunk', + 'c', + InputOption::VALUE_REQUIRED, + 'The number of CSV rows to handle at a time' + ) + ->addOption( + 'budget', + 'b', + InputOption::VALUE_REQUIRED, + 'How long, in seconds, to keep processing for' + ); + } + + // -------------------------------------------------------------------------- + + /** + * Executes the command + * + * @param InputInterface $oInput The Input Interface provided by Symfony + * @param OutputInterface $oOutput The Output Interface provided by Symfony + */ + protected function execute(InputInterface $oInput, OutputInterface $oOutput): int + { + parent::execute($oInput, $oOutput); + + try { + + $this->banner('User Import: Process'); + $this->recordLastRun(); + $this->processJobs(); + + } catch (Throwable $e) { + return $this->abort( + self::EXIT_CODE_FAILURE, + [$e->getMessage()] + ); + } + + $oOutput->writeln(''); + $oOutput->writeln('Complete!'); + + return self::EXIT_CODE_SUCCESS; + } + + // -------------------------------------------------------------------------- + + /** + * Records that the cron ran, so admin can warn when it stops + * + * @throws FactoryException + */ + protected function recordLastRun(): void + { + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + setAppSetting( + 'user-import-cron-last-run', + Constants::MODULE_SLUG, + $oNow->format('Y-m-d H:i:s') + ); + } + + // -------------------------------------------------------------------------- + + /** + * Works through the outstanding jobs, oldest first, until the budget is spent + * + * @throws FactoryException + * @throws ModelException + */ + protected function processJobs(): void + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Processor $oProcessor */ + $oProcessor = Factory::service('UserImportProcessor', Constants::MODULE_SLUG); + + $iChunk = (int) ($this->oInput->getOption('chunk') ?: $oProcessor::getChunkSize()); + $iBudget = (int) ($this->oInput->getOption('budget') ?: Config::get('AUTH_USER_IMPORT_RUN_BUDGET', static::RUN_BUDGET)); + $fDeadline = microtime(true) + $iBudget; + + $aSkip = []; + $iFound = 0; + + while (microtime(true) < $fDeadline) { + + $oImport = $this->getNextJob($aSkip); + if (empty($oImport)) { + break; + } + + $iFound++; + $aSkip[] = $oImport->id; + + $sToken = md5(uniqid((string) getmypid(), true)); + + if (!$oModel->claim($oImport->id, $sToken)) { + $this->oOutput->writeln(sprintf( + 'Import #%s is claimed by another process, skipping', + $oImport->id + )); + continue; + } + + $this->oOutput->writeln(sprintf( + 'Processing import #%s (%s)', + $oImport->id, + $oImport->status->value + )); + + try { + + while (!$oImport->status->isTerminal() && microtime(true) < $fDeadline) { + + $sBefore = $this->fingerprint($oImport); + $oImport = $oProcessor->process($oImport, $iChunk); + + $this->oOutput->writeln(sprintf( + '↳ %s: validated %s, processed %s of %s (%s ok, %s warned, %s errored)', + $oImport->status->value, + $oImport->validated_count, + $oImport->processed_count, + $oImport->row_count ?? 0, + $oImport->success_count, + $oImport->warning_count, + $oImport->error_count + )); + + if ($this->fingerprint($oImport) === $sBefore) { + /** + * Recorded on the job as well as printed: console output + * from a cron run is nobody's idea of a diagnostic, and + * an admin with no shell has to be able to see why this + * stopped. + */ + $oImport = $oProcessor->abandon( + $oImport, + 'no progress was made between two consecutive attempts' + ); + + $this->oOutput->writeln('↳ No progress was made, abandoning this job'); + break; + } + } + + $this->reportOutcome($oImport); + + } finally { + $oModel->release($oImport->id); + } + } + + if (empty($iFound)) { + $this->oOutput->writeln('Nothing to do'); + } + } + + // -------------------------------------------------------------------------- + + /** + * Prints why a job ended the way it did + * + * The job's stored error already carries the phase, the cause and the first + * of the failing lines, so it is printed rather than recomposed here; that + * way the console, the details modal and the notification email can never + * disagree with one another. + */ + protected function reportOutcome(Resource\User\Import $oImport): void + { + if ($oImport->status === Status::FAILED) { + $this->error(array_merge( + [sprintf('Import #%s failed', $oImport->id)], + array_values(array_filter( + explode("\n", (string) $oImport->error), + fn(string $sLine): bool => trim($sLine) !== '' + )) + )); + + } elseif ($oImport->status === Status::PARTIAL) { + /** + * A warning is a row whose account exists but whose follow-up did + * not happen, so it is named separately: the reader's next step for + * an errored row is to correct the CSV, and for a warned one it is + * to finish the account by hand. + */ + $this->warning(array_values(array_filter([ + sprintf( + 'Import #%s finished with %s of %s rows unresolved', + $oImport->id, + $oImport->error_count + $oImport->warning_count, + $oImport->row_count ?? 0 + ), + $oImport->error_count + ? sprintf('%s rows failed and no account was created', $oImport->error_count) + : null, + $oImport->warning_count + ? sprintf( + '%s accounts were created but something afterwards did not complete', + $oImport->warning_count + ) + : null, + ]))); + } + + if ($oImport->log_id) { + $this->oOutput->writeln(sprintf( + '↳ Log: CDN object #%s', + $oImport->log_id + )); + } + } + + // -------------------------------------------------------------------------- + + /** + * Returns the oldest unclaimed job which this runner owns + * + * @param int[] $aSkip Jobs which have already been looked at this run + * + * @throws FactoryException + * @throws ModelException + */ + protected function getNextJob(array $aSkip): ?Resource\User\Import + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $oDb + ->select('id') + ->where('runner', Runner::CRON->value) + ->where_in('status', Status::values(Status::active())) + ->where('claim_token', null) + ->order_by('id', 'asc') + ->limit(1); + + if (!empty($aSkip)) { + $oDb->where_not_in('id', $aSkip); + } + + $oRow = $oDb->get($oModel->getTableName())->row(); + + /** @var Resource\User\Import|null $oImport */ + $oImport = $oRow ? $oModel->getById((int) $oRow->id) : null; + + return $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * A cheap representation of the job's progress, used to detect a stall + */ + protected function fingerprint(Resource\User\Import $oImport): string + { + return implode(':', [ + $oImport->status->value, + $oImport->validated_count, + $oImport->processed_count, + ]); + } +} diff --git a/src/Cron/Task/User/Import/Clean.php b/src/Cron/Task/User/Import/Clean.php new file mode 100644 index 00000000..6aec2a25 --- /dev/null +++ b/src/Cron/Task/User/Import/Clean.php @@ -0,0 +1,39 @@ +query(<<query(<< $oStatus->value, $aStatuses); + } +} diff --git a/src/Exception/User/Import/LogException.php b/src/Exception/User/Import/LogException.php new file mode 100644 index 00000000..39cb75e6 --- /dev/null +++ b/src/Exception/User/Import/LogException.php @@ -0,0 +1,19 @@ +type('user_import_complete'); + } + + // -------------------------------------------------------------------------- + + /** + * Returns test data to use when sending test emails + * + * @return array + */ + public function getTestData(): array + { + return [ + 'import' => [ + 'row_count' => 100, + 'success' => 97, + 'warnings' => 1, + 'errors' => 2, + 'url' => siteUrl('admin/auth/import/preview/1'), + ], + /** + * Null on the happy path; complete() only writes an error here when + * the log itself could not be attached, or when the app's + * onImportComplete() hook objected. One block, so no phase line - + * see Service\User\Import\Processor::summariseError(). + */ + 'error' => null, + 'log_url' => siteUrl('admin/auth/import/log/1'), + ]; + } +} diff --git a/src/Factory/Email/User/Import/Failed.php b/src/Factory/Email/User/Import/Failed.php new file mode 100644 index 00000000..8bdfbfa7 --- /dev/null +++ b/src/Factory/Email/User/Import/Failed.php @@ -0,0 +1,51 @@ +type('user_import_failed'); + } + + // -------------------------------------------------------------------------- + + /** + * Returns test data to use when sending test emails + * + * @return array + */ + public function getTestData(): array + { + return [ + 'import' => [ + 'warnings' => 0, + 'errors' => 3, + 'url' => siteUrl('admin/auth/import/preview/1'), + ], + /** + * Summary and phase, which is all summariseError() lets through - + * the quoted rows and the cause stay in the log. Two blocks, because + * that is the shape the template has to be previewed in. + */ + 'error' => implode("\n\n", [ + '3 rows in the CSV could not be validated, so no user accounts were created. Correct them and upload the file again.', + 'Phase: validating the CSV (100 of 100 rows checked)', + ]), + 'log_url' => siteUrl('admin/auth/import/log/1'), + ]; + } +} diff --git a/src/Model/User/Import.php b/src/Model/User/Import.php new file mode 100644 index 00000000..7d8d13e6 --- /dev/null +++ b/src/Model/User/Import.php @@ -0,0 +1,312 @@ +hasOne('user', 'User', Constants::MODULE_SLUG, 'created_by') + ->hasOne('object', 'Object', Cdn\Constants::MODULE_SLUG) + ->hasOne('log', 'Object', Cdn\Constants::MODULE_SLUG); + } + + // -------------------------------------------------------------------------- + + /** + * Attempts to take exclusive ownership of a job + * + * The update only lands if the job is unclaimed and in a status a runner is + * permitted to pick up; the read back confirms it was this process which won. + * + * @param int $iId The job to claim + * @param string $sToken The token to claim it with + * + * @throws FactoryException + * @throws ModelException + */ + public function claim(int $iId, string $sToken): bool + { + $oDb = $this->getDb(); + + $oDb + ->set('claim_token', $sToken) + ->set('claimed', 'NOW()', false) + ->where('id', $iId) + ->where('claim_token', null) + ->where_in('status', Status::values(Status::active())) + ->update($this->getTableName()); + + return (bool) $oDb + ->where('id', $iId) + ->where('claim_token', $sToken) + ->count_all_results($this->getTableName()); + } + + // -------------------------------------------------------------------------- + + /** + * Relinquishes ownership of a job + * + * @throws FactoryException + * @throws ModelException + */ + public function release(int $iId): void + { + $oDb = $this->getDb(); + + $oDb + ->set('claim_token', null) + ->set('claimed', null) + ->where('id', $iId) + ->update($this->getTableName()); + } + + // -------------------------------------------------------------------------- + + /** + * Records how far through the validation phase the job is + * + * @throws FactoryException + * @throws ModelException + */ + public function setValidatedCount(int $iId, int $iCount): void + { + if (!$this->update($iId, ['validated_count' => $iCount])) { + throw new ModelException($this->lastError()); + } + } + + // -------------------------------------------------------------------------- + + /** + * Recalculates the error count from the item table + * + * The counts are derived rather than incremented so that a job which is + * interrupted and resumed cannot double count, or lose, a row. + * + * @throws FactoryException + * @throws ModelException + */ + public function syncValidationCounts(int $iId): void + { + $this->update($iId, [ + 'error_count' => $this->countItems($iId, ItemStatus::ERROR), + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Recalculates the processing cursor and counts from the item table + * + * @throws FactoryException + * @throws ModelException + */ + public function syncProcessingCounts(int $iId): void + { + $this->update($iId, [ + 'processed_count' => $this->countItems($iId), + 'success_count' => $this->countItems($iId, ItemStatus::SUCCESS), + 'warning_count' => $this->countItems($iId, ItemStatus::WARNING), + 'error_count' => $this->countItems($iId, ItemStatus::ERROR), + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Counts the job's items, optionally of a given status + * + * @throws FactoryException + * @throws ModelException + */ + protected function countItems(int $iId, ?ItemStatus $oStatus = null): int + { + /** @var Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + $oDb = $this->getDb(); + $oDb->where('import_id', $iId); + + if ($oStatus !== null) { + $oDb->where('status', $oStatus->value); + } + + return (int) $oDb->count_all_results($oItemModel->getTableName()); + } + + // -------------------------------------------------------------------------- + + /** + * Destroys the CDN objects belonging to a job + * + * Call this only *after* the job row has been deleted: `object_id` is + * ON DELETE CASCADE, so destroying the CSV while the row still exists takes + * the row with it - inside objectDestroy()'s own transaction - and the + * delete() which follows then reports the job as missing. + * + * Failures are collected rather than thrown: the row is already gone, so a + * file which outlives it is litter, not an error. Note that + * `auth:user:import:clean` walks job rows, so it will never reap these. + * objectDestroy() signals failure both by returning false - for a missing + * object, a driver failure, or a rolled back transaction - and by throwing, + * so both are handled. + * + * @return array The reason each object could not be destroyed, keyed by object ID + * + * @throws FactoryException + */ + public function destroyObjects(Resource\User\Import $oImport): array + { + $oCdn = $this->getCdn(); + $aErrors = []; + + foreach (array_filter([$oImport?->object?->id ?? $oImport->object_id, $oImport->log_id]) as $iObjectId) { + try { + if (!$oCdn->objectDestroy($iObjectId)) { + $aErrors[(int) $iObjectId] = $oCdn->lastError() ?: 'Unknown error'; + } + } catch (Throwable $e) { + $aErrors[(int) $iObjectId] = $e->getMessage(); + } + } + + return $aErrors; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes the job's items + * + * Called when a job is restarted from PENDING. The unique key on + * (import_id, line) is what makes resuming idempotent, so leaving the + * previous run's items in place would make every line look as though it had + * already been handled - and the counts, which are re-derived rather than + * incremented, would be taken from that stale run. + * + * @return bool Whether the items were deleted + * @throws FactoryException + */ + public function deleteItems(int $iId): bool + { + /** @var Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + return (bool) $this + ->getDb() + ->where('import_id', $iId) + ->delete($oItemModel->getTableName()); + } + + // -------------------------------------------------------------------------- + + /** + * Returns the database service + * + * Broken out so that the claim/release contract can be exercised in + * isolation, without standing up a database. + * + * @throws FactoryException + */ + protected function getDb(): Database + { + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + return $oDb; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the CDN service + * + * Broken out so that destroyObjects() can be exercised in isolation, as + * getDb() does for the database. + * + * @throws FactoryException + */ + protected function getCdn(): Cdn\Service\Cdn + { + /** @var Cdn\Service\Cdn $oCdn */ + $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + return $oCdn; + } +} diff --git a/src/Model/User/Import/Item.php b/src/Model/User/Import/Item.php new file mode 100644 index 00000000..470e087f --- /dev/null +++ b/src/Model/User/Import/Item.php @@ -0,0 +1,79 @@ +hasOne('user', 'User', Constants::MODULE_SLUG); + } +} diff --git a/src/Queue/Task/User/Import.php b/src/Queue/Task/User/Import.php new file mode 100644 index 00000000..667140f2 --- /dev/null +++ b/src/Queue/Task/User/Import.php @@ -0,0 +1,130 @@ +get(); + + /** @var Resource\User\Import|null $oImport */ + $oImport = $oModel->getById($iId); + + if (empty($oImport)) { + throw new RuntimeException(sprintf( + 'User import #%s does not exist', + $iId + )); + + } elseif ($oImport->status->isTerminal()) { + return; + } + + $sToken = md5(uniqid((string) getmypid(), true)); + + if (!$oModel->claim($iId, $sToken)) { + // Another process is already working this job + return; + } + + try { + + while (!$oImport->status->isTerminal()) { + + $sBefore = $this->fingerprint($oImport); + $oImport = $oProcessor->process($oImport); + + if ($this->fingerprint($oImport) === $sBefore) { + + /** + * Recorded on the job before it is thrown: module-queue's own + * failure record is not reachable from the user import admin, + * so without this an admin sees a job stuck mid-flight with + * nothing to explain it. + */ + $oProcessor->abandon( + $oImport, + 'no progress was made between two consecutive attempts' + ); + + throw new RuntimeException(sprintf( + 'User import #%s made no progress; abandoning to avoid spinning', + $iId + )); + } + } + + } finally { + $oModel->release($iId); + } + } + + // -------------------------------------------------------------------------- + + /** + * A cheap representation of the job's progress, used to detect a stall + */ + protected function fingerprint(Resource\User\Import $oImport): string + { + return implode(':', [ + $oImport->status->value, + $oImport->validated_count, + $oImport->processed_count, + ]); + } +} diff --git a/src/Resource/User/Import.php b/src/Resource/User/Import.php new file mode 100644 index 00000000..c106aaa6 --- /dev/null +++ b/src/Resource/User/Import.php @@ -0,0 +1,125 @@ +status = $resource->status instanceof Status + ? $resource->status + : Status::from($resource->status); + + $resource->runner = $resource->runner instanceof Runner || $resource->runner === null + ? $resource->runner + : Runner::tryFrom($resource->runner); + + if (!$resource->additional instanceof stdClass) { + $resource->additional = (object) (json_decode((string) $resource->additional) ?: []); + } + + $resource->skip_registered = (bool) ($resource->skip_registered ?? false); + $resource->warning_count = (int) ($resource->warning_count ?? 0); + + parent::__construct($resource, $model); + } + + // -------------------------------------------------------------------------- + + /** + * The percentage of the job which has been completed + */ + public function getPercent(): int + { + if (empty($this->row_count)) { + return 0; + + } elseif ($this->status->isTerminal()) { + return 100; + } + + $iCursor = $this->status === Status::VALIDATING + ? $this->validated_count + : $this->processed_count; + + return (int) min(100, floor(($iCursor / $this->row_count) * 100)); + } +} diff --git a/src/Resource/User/Import/Item.php b/src/Resource/User/Import/Item.php new file mode 100644 index 00000000..c0947ae9 --- /dev/null +++ b/src/Resource/User/Import/Item.php @@ -0,0 +1,33 @@ +status = $resource->status instanceof ItemStatus + ? $resource->status + : ItemStatus::from($resource->status); + + parent::__construct($resource, $model); + } +} diff --git a/src/Service/User/Import.php b/src/Service/User/Import.php index 52486bad..1aeef8a5 100644 --- a/src/Service/User/Import.php +++ b/src/Service/User/Import.php @@ -14,11 +14,14 @@ use Closure; use Nails\Auth\Constants; +use Nails\Auth\Exception\User\Import\TemplateException; use Nails\Auth\Model\User; +use Nails\Auth\Resource; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\NailsException; use Nails\Common\Exception\ValidationException; use Nails\Common\Factory\Model\Field; +use Nails\Common\Service\Database; use Nails\Common\Service\DateTime; use Nails\Common\Service\FormValidation; use Nails\Common\Validation\Context; @@ -32,6 +35,21 @@ */ class Import { + /** + * The number of values whichExist() puts in a single IN clause + * + * CodeIgniter's query builder runs a preg_match over the whole compiled + * condition, so one enormous IN clause fails outright with "regular + * expression is too large" - a 1,362 value list compiles to roughly 35KB, + * past what PCRE will accept. Chunking also keeps the statement well inside + * max_allowed_packet. Still one query per few hundred values rather than one + * per row, which is the point. + * + * @var int + */ + const WHICH_EXIST_CHUNK_SIZE = 250; + + /** * Returns the columns which can be imported * @@ -68,6 +86,85 @@ public function getUniqueKeys(): array ]; } + /** + * Returns which of the given values are already taken + * + * Batched on purpose: this is asked once per unique key for a whole file, or + * a whole chunk, rather than once per row. One indexed lookup answers it, + * where the per-row closures this replaced cost a query each. + * + * An app which adds a unique key must handle it here; there is deliberately + * no silent fallback, because a uniqueness check which quietly answers "none + * of them" is worse than one which fails. The key is checked before the + * empty-value short circuit so an unsupported key is caught even when there + * is nothing to look up. + * + * @param string $sKey The unique key being checked + * @param string[] $aValues The values to look for + * + * @return string[] The values which are already taken, lowercased + * @throws TemplateException If the key cannot be looked up + * @throws FactoryException + */ + public function whichExist(string $sKey, array $aValues): array + { + /** @var User\Email $oUserEmailModel */ + $oUserEmailModel = Factory::model('UserEmail', Constants::MODULE_SLUG); + /** @var User $oUserModel */ + $oUserModel = Factory::model('User', Constants::MODULE_SLUG); + + [$sTable, $sColumn] = match ($sKey) { + 'email' => [$oUserEmailModel->getTableName(), 'email'], + 'username' => [$oUserModel->getTableName(), 'username'], + default => throw new TemplateException(sprintf( + '"%s" is listed by %s::getUniqueKeys() but %s::whichExist() does not know how to ' + . 'look it up; override it to say where the value lives.', + $sKey, + static::class, + static::class + )), + }; + + $aValues = array_values( + array_unique( + array_filter( + array_map(fn($mValue): string => strtolower(trim((string) $mValue)), $aValues), + fn(string $sValue): bool => $sValue !== '' + ) + ) + ); + + if (empty($aValues)) { + return []; + } + + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aExisting = []; + + foreach (array_chunk($aValues, static::WHICH_EXIST_CHUNK_SIZE) as $aChunk) { + + /** + * Queried directly rather than through the model: this is an + * existence probe, and Model\User::getCountCommon() would hydrate + * every match across three joins to answer a question about a single + * column. + */ + $aRows = $oDb + ->select($sColumn) + ->where_in($sColumn, $aChunk) + ->get($sTable) + ->result(); + + foreach ($aRows as $oRow) { + $aExisting[] = strtolower(trim((string) $oRow->{$sColumn})); + } + } + + return $aExisting; + } + /** * Returns the validation rules for a given key * @@ -88,32 +185,23 @@ public function getValidationRules(string $key): array $oDateTimeService = Factory::service('DateTime'); return match ($key) { + /** + * Note that uniqueness is deliberately absent from here and from + * 'username'. It used to be a closure calling getByEmail() per row, + * which cost a query - two, on a hit - for every line of the file. + * It is now a batched whole-of-file concern; see whichExist() and + * Import\Validator::detectRegistered(). + */ 'email' => array_filter([ in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['EMAIL', 'BOTH']) ? FormValidation::RULE_REQUIRED : null, FormValidation::RULE_VALID_EMAIL, - function ($email) use ($oUserModel) { - if ($email && $oUserModel->getByEmail($email)) { - throw new ValidationException(sprintf( - '"%s" is already registered', - $email - )); - } - }, ]), 'username' => array_filter([ in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['USERNAME', 'BOTH']) ? FormValidation::RULE_REQUIRED : null, - function ($username) use ($oUserModel) { - if ($username && $oUserModel->getByUsername($username)) { - throw new ValidationException(sprintf( - '"%s" is already registered', - $username - )); - } - }, function ($username) use ($oUserModel) { if ($username && !$oUserModel->isValidUsername($username)) { throw new ValidationException(sprintf( @@ -253,4 +341,122 @@ public function parseAdditionalFields(string $sKey, mixed $mValue): mixed { return $mValue; } + + // -------------------------------------------------------------------------- + // Lifecycle hooks + // + // All five are no-ops here, and an override should call parent:: as + // whichExist() documents. Read the failure semantics before overriding: + // they are not the same for each hook, because what the processor can still + // do about a failure is not the same at each point. + // -------------------------------------------------------------------------- + + /** + * Fired once, when validation has passed and before the first account is created + * + * The one hook which may refuse the whole job: this runs while nothing has + * been created, so a throw here is deliberately left to reach the processor, + * which fails the job with the reason folded into its composed error. Throw + * a ValidationException with a sentence an admin can act on. + * + * Note that it fires once per job, not once per chunk - run() is called + * many times, across processes, but this is not. + * + * @param Resource\User\Import $oImport The job which is about to start + */ + public function onImportStart(Resource\User\Import $oImport): void + { + } + + // -------------------------------------------------------------------------- + + /** + * Fired for every row, immediately before the account is created + * + * The data is returned rather than taken by reference; whatever comes back + * is what reaches Model\User::create(), so an override which forgets to + * return imports nothing. Keys create() does not recognise - anything which + * is neither a user column nor a meta field - are silently discarded by it, + * so app-specific values belong in afterUserCreate() rather than here. + * + * This runs *after* the job's additional fields have been applied, so an + * override sees, and may overrule, what parseAdditionalFields() decided. + * + * A throw marks the row ERROR and leaves no account behind; the rest of the + * job carries on. + * + * @param array $aUserData The data create() will be given + * @param Resource\User\Import $oImport The job the row belongs to + * @param array $aRow The CSV row, aligned to the header, + * including columns create() will drop + * + * @return array The data to create the account with + */ + public function prepareUserData(array $aUserData, Resource\User\Import $oImport, array $aRow): array + { + return $aUserData; + } + + // -------------------------------------------------------------------------- + + /** + * Fired for every row, immediately after the account is created + * + * This is where app-specific follow-on work belongs - applying data a user + * column cannot hold, starting a subscription, and so on. + * + * The account is committed by the time this is called: Model\User::create() + * commits, fires USER_CREATED and queues the welcome email before it + * returns, so nothing here can undo it. A throw is therefore recorded + * rather than retried - the row is marked WARNING against the new account's + * ID, the exception is reported, and the job finishes PARTIAL so that a + * silent failure cannot pass as a clean import. + * + * @param Resource\User $oUser The account which was just created + * @param array $aUserData The data it was created with + * @param Resource\User\Import $oImport The job the row belongs to + */ + public function afterUserCreate( + Resource\User $oUser, + array $aUserData, + Resource\User\Import $oImport + ): void { + } + + // -------------------------------------------------------------------------- + + /** + * Fired once, when the job has finished + * + * Every account which was going to be created already exists, so this must + * not pretend the job can be undone. A throw is reported and appended to the + * job's stored error - which is what the details modal and the notification + * email show - but it cannot re-brand the job: a finished import stays + * COMPLETE or PARTIAL. + * + * @param Resource\User\Import $oImport The job, as it now stands + */ + public function onImportComplete(Resource\User\Import $oImport): void + { + } + + // -------------------------------------------------------------------------- + + /** + * Fired once, when the job has been rejected or abandoned + * + * Note that this fires for a job rejected during validation, where no + * account was ever created, as well as for one brought down part way + * through, where some exist; read `$oImport->success_count` rather than + * assuming either. + * + * A throw is reported and otherwise swallowed. It cannot be rethrown: the + * processor's failure path is reached from its own catch-all, so an + * exception escaping here would recurse straight back into it. + * + * @param Resource\User\Import $oImport The job, as it now stands + */ + public function onImportFailed(Resource\User\Import $oImport): void + { + } } diff --git a/src/Service/User/Import/Csv.php b/src/Service/User/Import/Csv.php new file mode 100644 index 00000000..c09cefa8 --- /dev/null +++ b/src/Service/User/Import/Csv.php @@ -0,0 +1,358 @@ +open($sPath); + + try { + $aHeader = $this->read($rHandle); + } finally { + fclose($rHandle); + } + + if ($this->isBlank($aHeader)) { + return []; + } + + return array_map(fn($sValue) => trim((string) $sValue), $aHeader); + } + + // -------------------------------------------------------------------------- + + /** + * Counts the number of data rows in the CSV, the header excluded + * + * @param string $sPath The path to the CSV + */ + public function countRows(string $sPath): int + { + $rHandle = $this->open($sPath); + + try { + + // Discard the header + $this->read($rHandle); + + $iCount = 0; + while (($aRow = $this->read($rHandle)) !== false) { + if (!$this->isBlank($aRow)) { + $iCount++; + } + } + + } finally { + fclose($rHandle); + } + + return $iCount; + } + + // -------------------------------------------------------------------------- + + /** + * Yields raw (i.e. unkeyed) data rows, keyed by their line number + * + * @param string $sPath The path to the CSV + * @param int $iOffset The number of data rows to skip + * @param int|null $iLimit The maximum number of rows to yield, null for all + * + * @return Generator + */ + public function readRawRows(string $sPath, int $iOffset = 0, ?int $iLimit = null): Generator + { + $rHandle = $this->open($sPath); + + try { + + // Discard the header + $this->read($rHandle); + + $iIndex = 0; + $iSent = 0; + + while (($aRow = $this->read($rHandle)) !== false) { + + if ($this->isBlank($aRow)) { + continue; + } + + $iIndex++; + + if ($iIndex <= $iOffset) { + continue; + + } elseif ($iLimit !== null && $iSent >= $iLimit) { + break; + } + + $iSent++; + + yield $iIndex + static::HEADER_LINE => $aRow; + } + + } finally { + fclose($rHandle); + } + } + + // -------------------------------------------------------------------------- + + /** + * Yields data rows keyed by line number, each combined with the header and + * with the import service's default values applied to blank cells + * + * @param string $sPath The path to the CSV + * @param int $iOffset The number of data rows to skip + * @param int|null $iLimit The maximum number of rows to yield, null for all + * + * @return Generator> + * @throws FactoryException + * @throws NailsException + */ + public function readRows(string $sPath, int $iOffset = 0, ?int $iLimit = null): Generator + { + $aHeader = $this->getHeader($sPath); + + foreach ($this->readRawRows($sPath, $iOffset, $iLimit) as $iLine => $aRow) { + yield $iLine => $this->applyDefaults( + $this->align($aHeader, $aRow) + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Keys a raw row by the header's columns + * + * Rows which are short are padded, rows which are long are truncated; the + * discrepancy is reported separately by the validator so that a malformed + * row produces a legible error rather than an exception. + * + * @param string[] $aHeader The CSV's header row + * @param string[] $aRow The raw row + * + * @return array + */ + public function align(array $aHeader, array $aRow): array + { + $aOut = []; + $aRow = array_values($aRow); + $iIndex = 0; + + foreach ($aHeader as $sKey) { + $aOut[$sKey] = trim((string) ($aRow[$iIndex] ?? '')); + $iIndex++; + } + + return $aOut; + } + + // -------------------------------------------------------------------------- + + /** + * Replaces blank cells with the import service's default value for that key + * + * @param array $aRow The keyed row + * + * @return array + * @throws FactoryException + * @throws NailsException + */ + public function applyDefaults(array $aRow): array + { + /** @var \Nails\Auth\Service\User\Import $oImportService */ + $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); + + foreach ($aRow as $sKey => $sValue) { + if ($sValue === '') { + $aRow[$sKey] = $oImportService->getDefaultValue($sKey); + } + } + + return $aRow; + } + + // -------------------------------------------------------------------------- + + /** + * Builds the import's log CSV + * + * The source CSV is streamed and joined, line by line, against the supplied + * items; this reproduces the log format the import has always emitted (the + * original columns, plus `id`, `status` and `message`) without ever having + * held a second copy of the file. + * + * @param string $sPath The path to the source CSV + * @param iterable $aItems Keyed by line + * number, ascending + * + * @return string The path to the generated log CSV + */ + public function writeLog(string $sPath, iterable $aItems): string + { + $oItems = $aItems instanceof Iterator + ? $aItems + : new \ArrayIterator(is_array($aItems) ? $aItems : iterator_to_array($aItems)); + + $oItems->rewind(); + + $aHeader = $this->getHeader($sPath); + $sLog = $this->getTempFile(); + $rLog = fopen($sLog, 'w'); + + if ($rLog === false) { + throw new RuntimeException(sprintf('Failed to open "%s" for writing', $sLog)); + } + + try { + + fputcsv($rLog, array_merge($aHeader, ['id', 'status', 'message']), escape: ''); + + foreach ($this->readRawRows($sPath) as $iLine => $aRow) { + + if (!$oItems->valid()) { + break; + + } elseif ((int) $oItems->key() !== $iLine) { + continue; + } + + $aItem = (array) $oItems->current(); + + fputcsv( + $rLog, + array_merge( + array_values($this->align($aHeader, $aRow)), + [ + $aItem['id'] ?? null, + $aItem['status'] ?? '', + $aItem['message'] ?? '', + ] + ), + escape: '' + ); + + $oItems->next(); + } + + } finally { + fclose($rLog); + } + + return $sLog; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the path to a new temporary file + */ + public function getTempFile(): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-user-import-'); + + if ($sPath === false) { + throw new RuntimeException('Failed to create a temporary file'); + } + + return $sPath; + } + + // -------------------------------------------------------------------------- + + /** + * Opens the CSV for reading + * + * @return resource + */ + protected function open(string $sPath) + { + $rHandle = @fopen($sPath, 'r'); + + if ($rHandle === false) { + throw new RuntimeException(sprintf( + 'Failed to open "%s" for reading', + $sPath + )); + } + + return $rHandle; + } + + // -------------------------------------------------------------------------- + + /** + * Reads a single record + * + * The empty escape character matches the `str_getcsv($line, escape: '')` + * semantics the import has always used; `SplFileObject` cannot be relied + * upon to honour it, hence the plain handle. + * + * @param resource $rHandle + * + * @return string[]|false + */ + protected function read($rHandle): array|false + { + return fgetcsv($rHandle, escape: ''); + } + + // -------------------------------------------------------------------------- + + /** + * Whether a record is an empty line + * + * @param string[]|false $aRow + */ + protected function isBlank(array|false $aRow): bool + { + return $aRow === false || $aRow === [null] || $aRow === ['']; + } +} diff --git a/src/Service/User/Import/Dispatcher.php b/src/Service/User/Import/Dispatcher.php new file mode 100644 index 00000000..0e23d919 --- /dev/null +++ b/src/Service/User/Import/Dispatcher.php @@ -0,0 +1,122 @@ +getRunner(); + + if (!$oModel->update($oImport->id, [ + 'runner' => $oRunner->value, + 'status' => Status::PENDING->value, + ])) { + throw new ModelException($oModel->lastError()); + } + + if ($oRunner === Runner::QUEUE) { + Factory::service('Manager', \Nails\Queue\Constants::MODULE_SLUG) + ->push( + new ImportTask(), + Factory::factory('Data', \Nails\Queue\Constants::MODULE_SLUG, $oImport->id) + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Whether the runner has checked in recently enough to be believed + * + * There is nothing to check for the queue runner; module-queue monitors its + * own workers, and a warning we cannot substantiate is worse than none. + * + * @throws FactoryException + */ + public function isRunning(): bool + { + if ($this->getRunner() === Runner::QUEUE) { + return true; + } + + $sLastRun = appSetting('user-import-cron-last-run', Constants::MODULE_SLUG); + if (empty($sLastRun)) { + return false; + } + + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + return ($oNow->getTimestamp() - (new \DateTime($sLastRun))->getTimestamp()) <= static::RUNNING_THRESHOLD; + } + + // -------------------------------------------------------------------------- + + /** + * Which runner will process jobs on this installation + */ + public function getRunner(): Runner + { + return Components::exists(static::QUEUE_MODULE) + ? Runner::QUEUE + : Runner::CRON; + } +} diff --git a/src/Service/User/Import/Processor.php b/src/Service/User/Import/Processor.php new file mode 100644 index 00000000..fd75d25b --- /dev/null +++ b/src/Service/User/Import/Processor.php @@ -0,0 +1,1674 @@ + 'import-user', + 'is_hidden' => true, + 'allowed_types' => 'csv', + ]; + + /** + * The number of rows to handle in a single call to process() + * + * @var int + */ + const CHUNK_SIZE = 100; + + /** + * The number of items to read from the database at a time when building the log + * + * @var int + */ + const LOG_PAGE_SIZE = 1000; + + /** + * The number of failing lines to quote in the job's error + * + * Enough to recognise a pattern - "they are all already registered" - without + * duplicating the log CSV, which carries every row. + * + * @var int + */ + const ERROR_SAMPLE_SIZE = 10; + + /** + * The prefix of the composed error's "how far did it get" fact + * + * A const because summariseError() has to recognise this line to keep it + * while dropping the facts either side of it; the two must not drift. + * + * @var string + */ + const ERROR_FACT_PHASE = 'Phase: '; + + /** + * The maximum length of the job's error + * + * The column is TEXT, so this is a sanity bound rather than a hard limit; it + * exists so a pathological CSV cannot fill the row with error text. + * + * @var int + */ + const ERROR_MAX_LENGTH = 8000; + + /** + * The maximum length of a stack trace recorded in the log + * + * @var int + */ + const ERROR_TRACE_MAX_LENGTH = 2000; + + /** + * The levels report() can log at + * + * @var string + */ + const LOG_INFO = 'info'; + const LOG_WARNING = 'warning'; + const LOG_ERROR = 'error'; + + // -------------------------------------------------------------------------- + + /** + * Advances the job by, at most, $iLimit rows + * + * @param Resource\User\Import $oImport The job to advance + * @param int|null $iLimit The number of rows to handle, null for the configured chunk size + * + * @return Resource\User\Import The job, as it now stands + * @throws FactoryException + * @throws ModelException + * @throws NailsException + */ + public function process(Resource\User\Import $oImport, ?int $iLimit = null): Resource\User\Import + { + $iLimit = $iLimit ?: static::getChunkSize(); + + try { + + return match ($oImport->status) { + Status::PENDING => $this->begin($oImport), + Status::VALIDATING => $this->validate($oImport, $iLimit), + Status::RUNNING => $this->run($oImport, $iLimit), + default => $oImport, + }; + + } catch (Throwable $e) { + /** + * Re-read before composing: the counts the message quotes have to be + * the ones on disk, not the ones this call started with. + */ + $oImport = $this->refresh($oImport); + + return $this->fail($oImport, $this->composeThrowableError($oImport, $e), $e); + } + } + + // -------------------------------------------------------------------------- + + /** + * Rejects a job which has stopped making progress + * + * Public because it is the runners, not the processor, which detect a stall - + * they are the ones holding the before-and-after fingerprints. Named to match + * the copy they already print rather than exposing fail() itself. + * + * @throws FactoryException + * @throws ModelException + */ + public function abandon(Resource\User\Import $oImport, string $sReason): Resource\User\Import + { + $oImport = $this->refresh($oImport); + + return $this->fail($oImport, $this->composeStallError($oImport, $sReason)); + } + + // -------------------------------------------------------------------------- + + /** + * The number of rows to handle in a single call to process() + */ + public static function getChunkSize(): int + { + return (int) Config::get('AUTH_USER_IMPORT_CHUNK', static::CHUNK_SIZE) ?: static::CHUNK_SIZE; + } + + // -------------------------------------------------------------------------- + + /** + * Opens the job: records when it started and how much work there is to do + * + * @throws FactoryException + * @throws ModelException + */ + protected function begin(Resource\User\Import $oImport): Resource\User\Import + { + $oModel = $this->getModel(); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + $sPath = $this->getSourcePath($oImport); + + $this->clearPreviousAttempt($oImport); + + $iRowCount = $oCsv->countRows($sPath); + + $oModel->update($oImport->id, [ + 'started' => $oNow->format('Y-m-d H:i:s'), + 'finished' => null, + 'log_id' => null, + 'row_count' => $iRowCount, + 'validated_count' => 0, + 'processed_count' => 0, + 'success_count' => 0, + 'warning_count' => 0, + 'error_count' => 0, + 'error' => null, + 'status' => Status::VALIDATING->value, + ]); + + $this->report( + $oImport, + sprintf( + 'starting; %s rows, %s runner', + number_format($iRowCount), + $oImport->runner?->value ?? 'unknown' + ), + null, + static::LOG_INFO + ); + + return $this->refresh($oImport); + } + + // -------------------------------------------------------------------------- + + /** + * Validates the next chunk of rows + * + * Nothing is created during this phase. If a single row fails then the whole + * job is rejected, because a half-imported file is worse than no import at + * all; the errors are written out as a log so they can be corrected and the + * file re-uploaded. + * + * @throws FactoryException + * @throws ModelException + * @throws NailsException + */ + protected function validate(Resource\User\Import $oImport, int $iLimit): Resource\User\Import + { + $oModel = $this->getModel(); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + + $sPath = $this->getSourcePath($oImport); + $aHeader = $oCsv->getHeader($sPath); + + /** + * Both are fatal and neither produces a per-row log worth generating: a + * template which cannot identify an account, or a header we do not + * understand, means no row could ever be imported. + */ + $oValidator->validateTemplate(); + $oValidator->validateHeader($aHeader); + + $aChunk = iterator_to_array( + $oCsv->readRawRows($sPath, $oImport->validated_count, $iLimit), + true + ); + + $aErrors = $oValidator->validateRows($aHeader, $aChunk); + + /** + * Whether an already-registered row is fatal is the admin's call, taken + * when the job was approved. When they have opted to skip, the rows are + * deliberately *not* recorded here and are left for run() to deal with + * as it reaches them - see the note there on why an item must not exist + * ahead of its row. + */ + if (!$oImport->skip_registered) { + foreach ($oValidator->detectRegistered($aHeader, $aChunk) as $iLine => $aLineErrors) { + $aErrors[$iLine] = array_merge($aErrors[$iLine] ?? [], $aLineErrors); + } + } + + foreach ($aErrors as $iLine => $aLineErrors) { + $this->recordItem( + $oImport, + $iLine, + ItemStatus::ERROR, + implode('; ', $aLineErrors) + ); + } + + $oModel->setValidatedCount($oImport->id, $oImport->validated_count + count($aChunk)); + $oModel->syncValidationCounts($oImport->id); + + $oImport = $this->refresh($oImport); + + // More to do? + if (!empty($aChunk) && $oImport->validated_count < $oImport->row_count) { + return $oImport; + } + + if ($oImport->error_count) { + + $this->report( + $oImport, + sprintf( + 'validation rejected %s of %s rows', + number_format($oImport->error_count), + number_format((int) $oImport->row_count) + ), + null, + static::LOG_WARNING + ); + + return $this->fail($oImport, $this->composeValidationError($oImport)); + } + + /** + * The app's last chance to refuse the job, and the only hook which is + * allowed to: nothing has been created yet, so a throw here is + * deliberately left to reach process()'s catch, which fails the job with + * the reason folded into its composed error. Every other hook is + * wrapped, because by the time they run there are accounts which cannot + * be taken back. + */ + $this->getImportService()->onImportStart($oImport); + + $oModel->update($oImport->id, [ + 'processed_count' => 0, + 'success_count' => 0, + 'warning_count' => 0, + 'error_count' => 0, + 'status' => Status::RUNNING->value, + ]); + + return $this->refresh($oImport); + } + + // -------------------------------------------------------------------------- + + /** + * Creates the users for the next chunk of rows + * + * @throws FactoryException + * @throws ModelException + * @throws NailsException + */ + protected function run(Resource\User\Import $oImport, int $iLimit): Resource\User\Import + { + $oModel = $this->getModel(); + $oUserModel = $this->getUserModel(); + $oImportService = $this->getImportService(); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + + $sPath = $this->getSourcePath($oImport); + $aHeader = $oCsv->getHeader($sPath); + $aKeys = $oImportService->getKeys(); + + $aChunk = iterator_to_array( + $oCsv->readRawRows($sPath, $oImport->processed_count, $iLimit), + true + ); + + /** + * One batched lookup for the chunk, and only when the admin asked for it: + * without their consent a row which became registered since validation + * is left to create(), which refuses it on its own authority and reports + * its own reason. + */ + $aRegistered = $oImport->skip_registered + ? $oValidator->detectRegistered($aHeader, $aChunk) + : []; + + /** + * A job which was interrupted mid-chunk will have a cursor which lags the + * item table; skip anything already accounted for so a row can never be + * imported twice. + */ + $aDone = $this->getHandledLines($oImport, array_keys($aChunk)); + + foreach ($aChunk as $iLine => $aRow) { + + if (in_array($iLine, $aDone, true)) { + continue; + } + + $aDatum = $oCsv->align($aHeader, $aRow); + + /** + * Claim the line before doing anything which cannot be undone; if the + * process dies part way through, the line is marked as handled and + * will not be attempted again. + */ + $iItemId = $this->recordItem( + $oImport, + $iLine, + ItemStatus::ERROR, + 'The import was interrupted before this row was completed' + ); + + if (empty($iItemId)) { + continue; + } + + /** + * Claimed, then resolved without creating anything. Handling the skip + * here rather than pre-recording it during validation is deliberate: + * the read offset above is processed_count, which only tracks the + * file because every row of a chunk yields exactly one item. An item + * which exists ahead of its row inflates the count and silently + * steps over a row which was never imported. + */ + if (array_key_exists($iLine, $aRegistered)) { + $this->resolveItem( + $iItemId, + ItemStatus::ERROR, + sprintf('%s; skipped', implode('; ', $aRegistered[$iLine])) + ); + continue; + } + + $bSendEmail = stringToBoolean($aDatum['send_email'] ?? false); + $aUserData = []; + + foreach ($aKeys as $sKey) { + $aUserData[$sKey] = $aDatum[$sKey] ?? null; + if ($aUserData[$sKey] === '') { + $aUserData[$sKey] = $oImportService->getDefaultValue($sKey); + } + } + + // Apply additional fields + foreach ($oImport->additional as $sProperty => $mValue) { + $aUserData[$sProperty] = $oImportService->parseAdditionalFields($sProperty, $mValue); + } + + try { + + /** + * Deliberately after the loop above rather than in place of it, + * so an app which overrides the hook without calling parent:: + * still gets the additional fields it configured. The raw row is + * handed over too, because it carries the columns create() is + * about to discard. + */ + $aUserData = $oImportService->prepareUserData($aUserData, $oImport, $aDatum); + + $oUser = $oUserModel->create($aUserData, $bSendEmail); + + if (!$oUser) { + $this->resolveItem( + $iItemId, + ItemStatus::ERROR, + /** + * lastError() returns false, not '', for an empty error + * stack; without the fallback a failed create() leaves + * the row with no reason recorded against it at all. + */ + $oUserModel->lastError() ?: 'The account could not be created' + ); + continue; + } + + /** + * The account is committed from here on - create() commits, and + * queues the welcome email, before it returns - so nothing below + * may pretend it can be undone. A hook which fails leaves the row + * WARNING against the new account's ID, which is the thread back + * to whatever an admin now has to finish by hand. + */ + try { + $oImportService->afterUserCreate($oUser, $aUserData, $oImport); + $this->resolveItem($iItemId, ItemStatus::SUCCESS, null, $oUser->id); + + } catch (Throwable $e) { + $this->resolveItem( + $iItemId, + ItemStatus::WARNING, + sprintf('The account was created, but %s', $e->getMessage()), + $oUser->id + ); + $this->report( + $oImport, + sprintf('line %s: afterUserCreate() failed', $iLine), + $e + ); + } + + } catch (Throwable $e) { + // Covers prepareUserData() and create() itself; no account exists + $this->resolveItem($iItemId, ItemStatus::ERROR, $e->getMessage()); + $this->report($oImport, sprintf('line %s could not be imported', $iLine), $e); + } + } + + $oModel->syncProcessingCounts($oImport->id); + $oImport = $this->refresh($oImport); + + if (empty($aChunk) || $oImport->processed_count >= $oImport->row_count) { + return $this->complete($oImport); + } + + return $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * Finishes the job, generating and attaching the log + * + * @throws FactoryException + * @throws ModelException + * @throws NailsException + */ + protected function complete(Resource\User\Import $oImport): Resource\User\Import + { + $oModel = $this->getModel(); + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + $aLog = $this->attachLog($oImport); + + // A warned account exists, so it is one of the accounts this created + $iCreated = $oImport->success_count + $oImport->warning_count; + + /** + * A log which could not be written must not turn a finished import into + * a failed one - the accounts exist either way - but nor should it + * disappear, so it is recorded against the job. This used to be an + * un-caught call, which meant a log failure here reached process()'s + * catch and re-branded a perfectly good import as FAILED. + */ + $sError = $aLog['error'] + ? $this->truncateError($this->appendLogFailure( + sprintf( + '%s user %s created; the import itself finished normally.', + number_format($iCreated), + $iCreated === 1 ? 'account was' : 'accounts were' + ), + $aLog['error'] + )) + : null; + + /** + * A warning is a row whose account exists but whose follow-up did not + * happen, so it counts against a clean finish just as an error does; a + * job which quietly failed every subscription must not read COMPLETE. + */ + $sStatus = $oImport->error_count || $oImport->warning_count + ? Status::PARTIAL->value + : Status::COMPLETE->value; + + if (!$oModel->update($oImport->id, [ + 'log_id' => $aLog['log_id'], + 'finished' => $oNow->format('Y-m-d H:i:s'), + 'error' => $sError, + 'status' => $sStatus, + ])) { + $this->report($oImport, sprintf( + 'failed to record completion; %s', + $oModel->lastError() ?: 'no reason was reported' + )); + } + + $oImport = $this->refresh($oImport); + + /** + * Fired before the notification, so a hook which refuses the finished + * job has its reason in the email as well as in the modal. + */ + $oImport = $this->fireImportComplete($oImport); + + $this->report( + $oImport, + sprintf( + '%s; %s created, %s warned, %s errored, log %s', + $sStatus, + number_format($oImport->success_count), + number_format($oImport->warning_count), + number_format($oImport->error_count), + $oImport->log_id ? '#' . $oImport->log_id : 'not attached' + ), + null, + static::LOG_INFO + ); + + $this->notify( + $oImport, + Factory::factory('EmailUserImportComplete', Constants::MODULE_SLUG) + ); + + return $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * Rejects the job + * + * @throws FactoryException + * @throws ModelException + */ + protected function fail( + Resource\User\Import $oImport, + string $sError, + ?Throwable $e = null + ): Resource\User\Import { + + $oModel = $this->getModel(); + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + /** + * Warnings are logged alongside the errors: a job which warned and then + * failed would otherwise drop the accounts which need attention from the + * only record which names them. + */ + $aLog = $oImport->error_count || $oImport->warning_count + ? $this->attachLog($oImport, [ItemStatus::ERROR, ItemStatus::WARNING]) + : ['log_id' => $oImport->log_id, 'error' => null]; + + if ($aLog['error']) { + $sError = $this->appendLogFailure($sError, $aLog['error']); + } + + $sError = $this->truncateError($sError); + + /** + * Reported rather than thrown: throwing here would re-enter process()'s + * catch and recurse straight back into fail(). Left alone, the job's + * fingerprint does not move and the runner's stall path picks it up. + */ + if (!$oModel->update($oImport->id, [ + 'log_id' => $aLog['log_id'], + 'finished' => $oNow->format('Y-m-d H:i:s'), + 'error' => $sError, + 'status' => Status::FAILED->value, + ])) { + $this->report($oImport, sprintf( + 'failed to record the failure; %s', + $oModel->lastError() ?: 'no reason was reported' + )); + } + + $this->report($oImport, sprintf('FAILED; %s', $this->firstLine($sError)), $e); + + $oImport = $this->refresh($oImport); + + $this->fireImportFailed($oImport); + + $this->notify( + $oImport, + Factory::factory('EmailUserImportFailed', Constants::MODULE_SLUG) + ); + + return $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * Emails the person who requested the import + * + * Sent rather than queued: this is the operator's feedback on the job, and + * queueing it would put it behind the welcome email of every row the import + * created. One inline send at the end of a finished job is a price worth + * paying for them finding out promptly whether it worked. + * + * Counts, links and a redacted error only. The failing rows quote cell + * values straight out of the CSV - an email address, a name - and an inbox + * is the wrong place to keep those. Whoever needs to know which rows failed + * opens the log, which is behind a permission check. + */ + protected function notify(Resource\User\Import $oImport, Complete|Failed $oEmail): void + { + $iRecipient = $oImport->created_by instanceof \Nails\Common\Resource + ? $oImport->created_by->id + : $oImport->created_by; + + if (empty($iRecipient)) { + return; + } + + try { + + $oEmail + ->to($iRecipient) + ->data([ + 'import' => [ + 'row_count' => $oImport->row_count, + 'success' => $oImport->success_count, + 'warnings' => $oImport->warning_count, + 'errors' => $oImport->error_count, + 'url' => siteUrl('admin/auth/import/preview/' . $oImport->id), + ], + 'error' => $this->summariseError($oImport->error), + /** + * The guarded admin route rather than the CDN object: the + * log carries personal data, so the URL which reaches it is + * minted behind a permission check and expires. + */ + 'log_url' => $oImport->log_id + ? siteUrl('admin/auth/import/log/' . $oImport->id) + : null, + ]) + ->send(); + + } catch (Throwable $e) { + /** + * An email which cannot be sent must not undo the import. The + * exception is deliberately not handed to the error handler: a bad + * mail configuration would otherwise raise an alert on every import. + */ + $this->report( + $oImport, + sprintf('could not send the notification email; %s', $e->getMessage()), + null, + static::LOG_WARNING + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Generates the log CSV, puts it in the CDN, and returns its object ID + * + * The ID is returned rather than a resource deliberately. Cdn::objectCreate() + * hands back a plain stdClass - Cdn::getObject() is declared `bool|stdClass` + * and never returns a Cdn\Resource\CdnObject - so declaring a resource + * return type here raises a TypeError on the way out, which the caller then + * has to know to catch. An int cannot be got wrong, and both callers only + * ever read the ID. The admin upload path converts via the Object model + * instead; see Admin\Import::uploadCsv(). + * + * @param iterable $aItems + * + * @return int The CDN object ID of the stored log + * @throws LogException If the log cannot be built or stored + */ + protected function uploadLog(Resource\User\Import $oImport, iterable $aItems): int + { + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + $oCdn = $this->getCdn(); + + try { + $sLogPath = $oCsv->writeLog($this->getSourcePath($oImport), $aItems); + + } catch (Throwable $e) { + throw new LogException( + sprintf('Failed to build the import log; %s', $e->getMessage()), + 0, + $e + ); + } + + try { + + $mLog = $oCdn->objectCreate( + $sLogPath, + static::IMPORT_BUCKET, + [ + 'no-md5-check' => true, + 'Content-Type' => 'text/csv', + 'filename_display' => sprintf( + 'user-import-log-%s.csv', + $oNow->format('Y-m-d_H-i-s') + ), + 'metadata' => $this->getLogMetaData($oImport), + ] + ); + + } finally { + @unlink($sLogPath); + } + + /** + * objectCreate() signals failure by returning false and stashing the + * reason on the service, so it has to be read out explicitly or it is + * lost. lastError() returns false - not '' - for an empty error stack, + * hence the fallback. + */ + if (empty($mLog) || empty($mLog->id)) { + throw new LogException(sprintf( + 'Failed to store the import log in the CDN; %s', + $oCdn->lastError() ?: 'no reason was reported' + )); + } + + return (int) $mLog->id; + } + + // -------------------------------------------------------------------------- + + /** + * Attaches the log to the job, keeping hold of the reason if it cannot be + * + * The single place a log is attached. A log which cannot be written must not + * mask the reason the job finished the way it did - but it must not vanish + * either, which is what used to happen. Throwable rather than LogException + * on purpose: a programming error in here must still not take the job down, + * but it must be reported. + * + * @param ItemStatus[]|null $aStatuses The statuses to log, null for every item + * + * @return array{log_id: int|null, error: string|null} + * @throws FactoryException + * @throws ModelException + */ + protected function attachLog(Resource\User\Import $oImport, ?array $aStatuses = null): array + { + try { + return [ + 'log_id' => $this->uploadLog($oImport, $this->streamItems($oImport, $aStatuses)), + 'error' => null, + ]; + + } catch (Throwable $e) { + $this->report($oImport, 'failed to attach the log', $e); + + return [ + 'log_id' => $oImport->log_id, + 'error' => $e->getMessage(), + ]; + } + } + + // -------------------------------------------------------------------------- + + /** + * Records something noteworthy about a job + * + * Everything goes to the application log prefixed with the job's ID, so + * `grep 'import #6' application/logs/log-*.php` is the recovery path for a + * job whose stored error is not enough. A per-job log file would be tidier + * but nothing in the UI could link to it, which is the very problem this is + * meant to solve. + */ + protected function report( + Resource\User\Import $oImport, + string $sMessage, + ?Throwable $e = null, + string $sLevel = self::LOG_ERROR + ): void { + + $sLine = sprintf('User import #%s: %s', $oImport->id, $sMessage); + + if ($e !== null) { + $sLine .= ' ' . json_encode($this->errorPayload($e), JSON_UNESCAPED_SLASHES); + } + + try { + match ($sLevel) { + static::LOG_INFO => $this->getLogger()->info($sLine), + static::LOG_WARNING => $this->getLogger()->warning($sLine), + default => $this->getLogger()->error($sLine), + }; + } catch (Throwable $eLogger) { + // A worker which cannot log must still finish the job + } + + if ($e === null || $sLevel !== static::LOG_ERROR) { + return; + } + + /** + * Hand the exception to whatever the app reports errors with - Sentry, + * Rollbar - without halting; see module-cron's Run::logException(). + */ + try { + $sDriver = Factory::service('ErrorHandler')::getDriverClass(); + $sDriver::exception($e, false); + } catch (Throwable $eHandler) { + // A reporting failure must never take the job with it + } + } + + // -------------------------------------------------------------------------- + + /** + * Reduces a Throwable to the detail worth keeping + * + * Mirrors the shape module-queue records for a failed job; see + * Nails\Queue\Service\Manager::buildErrorPayload(). Deliberately duplicated + * rather than called: that method is protected, and module-queue is only a + * dev/suggested dependency, so a hard reference would break every install + * without it. + * + * @return array{type: string, message: string, code: mixed, file: string, line: int, trace: string, occurred_at: string} + */ + protected function errorPayload(Throwable $e): array + { + $sTrace = $e->getTraceAsString(); + + if (mb_strlen($sTrace) > static::ERROR_TRACE_MAX_LENGTH) { + $sTrace = mb_substr($sTrace, 0, static::ERROR_TRACE_MAX_LENGTH) . '…'; + } + + return [ + 'type' => $e::class, + 'message' => $e->getMessage(), + 'code' => $e->getCode(), + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'trace' => $sTrace, + 'occurred_at' => (new \DateTime())->format('Y-m-d H:i:s'), + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Yields the job's items, in line order, in pages + * + * Takes a set of statuses rather than one: a failed job wants its errors and + * its warnings in the same log, and asking twice would interleave two sorted + * streams for no gain. + * + * @param ItemStatus[]|null $aStatuses The statuses to yield, null for every item + * + * @return Generator + * @throws FactoryException + * @throws ModelException + */ + protected function streamItems(Resource\User\Import $oImport, ?array $aStatuses = null): Generator + { + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $iOffset = 0; + + do { + + $oDb + ->select('line, user_id, status, message') + ->where('import_id', $oImport->id) + ->order_by('line', 'asc') + ->limit(static::LOG_PAGE_SIZE, $iOffset); + + if (!empty($aStatuses)) { + $oDb->where_in('status', array_map( + fn(ItemStatus $oStatus): string => $oStatus->value, + $aStatuses + )); + } + + $aRows = $oDb->get($oItemModel->getTableName())->result(); + + foreach ($aRows as $oRow) { + yield (int) $oRow->line => [ + 'id' => $oRow->user_id === null ? null : (int) $oRow->user_id, + 'status' => $oRow->status, + 'message' => $oRow->message, + ]; + } + + $iOffset += static::LOG_PAGE_SIZE; + + } while (count($aRows) === static::LOG_PAGE_SIZE); + } + + // -------------------------------------------------------------------------- + + /** + * Returns the lines, within the given set, which already have an item + * + * @param int[] $aLines + * + * @return int[] + * @throws FactoryException + * @throws ModelException + */ + protected function getHandledLines(Resource\User\Import $oImport, array $aLines): array + { + if (empty($aLines)) { + return []; + } + + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aRows = $oDb + ->select('line') + ->where('import_id', $oImport->id) + ->where_in('line', $aLines) + ->get($oItemModel->getTableName()) + ->result(); + + return array_map(fn($oRow) => (int) $oRow->line, $aRows); + } + + // -------------------------------------------------------------------------- + + /** + * Records the outcome of a line + * + * The unique key on (import_id, line) is what makes resuming idempotent; a + * clash simply means an earlier run already dealt with this line. + * + * @return int|null The new item's ID, or null if the line was already handled + * @throws FactoryException + */ + protected function recordItem( + Resource\User\Import $oImport, + int $iLine, + ItemStatus $oStatus, + ?string $sMessage = null, + ?int $iUserId = null + ): ?int { + + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + try { + + $iItemId = $oItemModel->create([ + 'import_id' => $oImport->id, + 'line' => $iLine, + 'user_id' => $iUserId, + 'status' => $oStatus->value, + 'message' => $sMessage, + ]); + + } catch (Throwable $e) { + /** + * A clash on the (import_id, line) unique key means an earlier run + * already dealt with this line, which is how resuming stays + * idempotent - the expected path, not a fault. Anything else means + * the row will silently never be imported, which is worth the one + * extra query to tell apart. + */ + if (empty($this->getHandledLines($oImport, [$iLine]))) { + $this->report($oImport, sprintf('line %s could not be recorded', $iLine), $e); + } + + return null; + } + + return $iItemId ?: null; + } + + // -------------------------------------------------------------------------- + + /** + * Updates a previously claimed line with what actually happened + * + * @throws FactoryException + * @throws ModelException + */ + protected function resolveItem( + int $iItemId, + ItemStatus $oStatus, + ?string $sMessage = null, + ?int $iUserId = null + ): void { + + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + $oItemModel->update($iItemId, [ + 'status' => $oStatus->value, + 'message' => $sMessage, + 'user_id' => $iUserId, + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Clears down a previous attempt at the job + * + * A job restarted from PENDING has to be cleared first: the unique key on + * (import_id, line) would otherwise make every line look as though it had + * already been handled, and the counts would be re-derived from the previous + * run's items. + * + * @throws FactoryException + * @throws ModelException + */ + protected function clearPreviousAttempt(Resource\User\Import $oImport): void + { + $this->getModel()->deleteItems($oImport->id); + + if (empty($oImport->log_id)) { + return; + } + + /** + * begin() is about to null log_id, and auth:user:import:clean only reaps + * objects it can still reach from a job row, so the previous log is + * destroyed here or not at all. + */ + try { + if (!$this->getCdn()->objectDestroy($oImport->log_id)) { + $this->report( + $oImport, + sprintf( + 'failed to destroy the previous log #%s; %s', + $oImport->log_id, + $this->getCdn()->lastError() ?: 'no reason was reported' + ), + null, + static::LOG_WARNING + ); + } + + } catch (Throwable $e) { + $this->report( + $oImport, + sprintf('failed to destroy the previous log #%s', $oImport->log_id), + $e + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Composes the job's error + * + * Plain text with newlines. Line 1 is the only line a non-technical admin + * has to read - it is what the list cell and the preview alert show - and + * everything below it is for whoever has to fix the CSV. + * + * Deliberately not translated: this is a stored audit record written by a + * worker whose locale is the app default rather than the reader's, so it is + * written in English and translated at presentation if it ever needs to be. + * + * @param array $aSample + */ + protected function composeError( + string $sSummary, + ?string $sPhase = null, + ?string $sCause = null, + ?string $sWhere = null, + array $aSample = [], + int $iTotal = 0 + ): string { + + $aBlocks = [$sSummary]; + + $aFacts = array_filter([ + $sPhase !== null ? static::ERROR_FACT_PHASE . $sPhase : null, + $sCause !== null ? 'Cause: ' . $sCause : null, + $sWhere !== null ? 'Where: ' . $sWhere : null, + ]); + + if (!empty($aFacts)) { + $aBlocks[] = implode("\n", $aFacts); + } + + if (!empty($aSample)) { + + $aLines = [ + $iTotal > count($aSample) + ? sprintf( + 'Rows with errors (showing the first %s of %s):', + number_format(count($aSample)), + number_format($iTotal) + ) + : sprintf('Rows with errors (%s):', number_format(count($aSample))), + ]; + + foreach ($aSample as $aItem) { + $aLines[] = sprintf( + ' Line %s: %s', + $aItem['line'], + $aItem['message'] ?: '(no reason was recorded)' + ); + } + + $aBlocks[] = implode("\n", $aLines); + } + + return implode("\n\n", $aBlocks); + } + + // -------------------------------------------------------------------------- + + /** + * Composes the error for a job rejected during validation + * + * @throws FactoryException + * @throws ModelException + */ + protected function composeValidationError(Resource\User\Import $oImport): string + { + return $this->composeError( + sprintf( + '%s %s in the CSV could not be validated, so no user accounts were created. Correct them and upload the file again.', + number_format($oImport->error_count), + $oImport->error_count === 1 ? 'row' : 'rows' + ), + $this->describePhase($oImport), + null, + null, + $this->sampleErrors($oImport, static::ERROR_SAMPLE_SIZE), + $oImport->error_count + ); + } + + // -------------------------------------------------------------------------- + + /** + * Composes the error for a job brought down by something unexpected + * + * @throws FactoryException + * @throws ModelException + */ + protected function composeThrowableError(Resource\User\Import $oImport, Throwable $e): string + { + $sSummary = 'The import stopped unexpectedly and did not finish.'; + + $sSurvivors = $this->describeSurvivors($oImport); + if ($sSurvivors !== null) { + $sSummary .= ' ' . $sSurvivors; + } + + return $this->composeError( + $sSummary, + $this->describePhase($oImport), + sprintf('%s - %s', $e::class, $e->getMessage()), + /** + * basename() only - an absolute path from inside a container is + * noise to an admin, and this string reaches their inbox. + */ + sprintf('%s line %s', basename($e->getFile()), $e->getLine()), + $this->sampleErrors($oImport, static::ERROR_SAMPLE_SIZE), + $oImport->error_count + ) . "\n\nThe full technical detail was written to the application log."; + } + + // -------------------------------------------------------------------------- + + /** + * Composes the error for a job which stopped making progress + * + * @throws FactoryException + * @throws ModelException + */ + protected function composeStallError(Resource\User\Import $oImport, string $sReason): string + { + $sSummary = 'The import made no progress and was stopped to prevent it looping.'; + + $sSurvivors = $this->describeSurvivors($oImport); + if ($sSurvivors !== null) { + $sSummary .= ' ' . $sSurvivors; + } + + return $this->composeError( + $sSummary, + $this->describePhase($oImport), + $sReason, + null, + $this->sampleErrors($oImport, static::ERROR_SAMPLE_SIZE), + $oImport->error_count + ); + } + + // -------------------------------------------------------------------------- + + /** + * Describes what survived a job which stopped part way through + * + * validate()'s "no user accounts were created" cannot be reused once run() + * has started: accounts may well exist by then, and telling an admin none + * were created when some were is worse than saying nothing at all. + */ + protected function describeSurvivors(Resource\User\Import $oImport): ?string + { + /** + * Warnings count as created: the account exists, it is only the work + * which was meant to follow it which did not happen. + */ + $iCreated = $oImport->success_count + $oImport->warning_count; + + if ($iCreated) { + return sprintf( + '%s user %s had already been created and %s been kept.', + number_format($iCreated), + $iCreated === 1 ? 'account' : 'accounts', + $iCreated === 1 ? 'has' : 'have' + ); + + } elseif ($oImport->status === Status::VALIDATING) { + return 'No user accounts were created.'; + } + + return null; + } + + // -------------------------------------------------------------------------- + + /** + * Describes how far the job had got + */ + protected function describePhase(Resource\User\Import $oImport): string + { + return match ($oImport->status) { + Status::PENDING => 'opening the job', + Status::VALIDATING => sprintf( + 'validating the CSV (%s of %s rows checked)', + number_format($oImport->validated_count), + number_format((int) $oImport->row_count) + ), + Status::RUNNING => sprintf( + 'creating user accounts (%s of %s rows processed)', + number_format($oImport->processed_count), + number_format((int) $oImport->row_count) + ), + default => sprintf('finishing the job (%s)', $oImport->status->value), + }; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the job's first failing lines + * + * The log CSV carries every row, but it sits behind a download; this is what + * the modal, the CLI and the database itself show, so it is worth one + * bounded query. Deliberately not the email - see notify(). + * + * @return array + * @throws FactoryException + * @throws ModelException + */ + protected function sampleErrors(Resource\User\Import $oImport, int $iLimit): array + { + return $this->sampleItems($oImport, ItemStatus::ERROR, $iLimit); + } + + + // -------------------------------------------------------------------------- + + /** + * Returns the job's first rows of a given status + * + * @return array + * @throws FactoryException + * @throws ModelException + */ + protected function sampleItems(Resource\User\Import $oImport, ItemStatus $oStatus, int $iLimit): array + { + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aRows = $oDb + ->select('line, message') + ->where('import_id', $oImport->id) + ->where('status', $oStatus->value) + ->order_by('line', 'asc') + ->limit($iLimit) + ->get($oItemModel->getTableName()) + ->result(); + + return array_map( + fn($oRow): array => [ + 'line' => (int) $oRow->line, + 'message' => $oRow->message, + ], + $aRows + ); + } + + // -------------------------------------------------------------------------- + + /** + * Reduces a composed error to the part which is safe to email + * + * Keeps the summary and the phase; drops the quoted rows, the exception and + * the file it came from. The rows name people and the cause can quote one + * too - a driver reporting a duplicate key repeats the address - so none of + * it belongs in an inbox. The stored error keeps everything; this is only + * what the notification is allowed to repeat. + */ + protected function summariseError(?string $sError): ?string + { + if (empty($sError)) { + return null; + } + + $aBlocks = explode("\n\n", $sError); + $sSummary = array_shift($aBlocks); + + $aPhase = array_filter( + explode("\n", implode("\n", $aBlocks)), + fn(string $sLine): bool => str_starts_with($sLine, static::ERROR_FACT_PHASE) + ); + + return implode("\n\n", [$sSummary, ...$aPhase]); + } + + // -------------------------------------------------------------------------- + + /** + * Notes, on the end of the job's error, that the log could not be attached + */ + protected function appendLogFailure(string $sError, string $sReason): string + { + return sprintf( + "%s\n\nThe error log could not be attached: %s", + $sError, + $sReason + ); + } + + // -------------------------------------------------------------------------- + + /** + * Bounds the job's error + * + * The column is TEXT, so this guards against a pathological CSV rather than + * against the schema. + */ + protected function truncateError(string $sError): string + { + /** + * Multibyte-aware: the composed error quotes cell values straight out of + * the CSV, and cutting one of those mid-character would leave invalid + * UTF-8 in the column. + */ + return mb_strlen($sError) > static::ERROR_MAX_LENGTH + ? mb_substr($sError, 0, static::ERROR_MAX_LENGTH) . '…' + : $sError; + } + + // -------------------------------------------------------------------------- + + /** + * The summary line of a composed error, for places with room for one line + */ + protected function firstLine(string $sError): string + { + return explode("\n", $sError, 2)[0]; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the local path of the job's source CSV + * + * @throws FactoryException + * @throws ValidationException + */ + protected function getSourcePath(Resource\User\Import $oImport): string + { + $sPath = $this->getCdn()->objectLocalPath($oImport->object_id); + + if (empty($sPath)) { + throw new ValidationException( + 'Failed to get a local path for the CSV file' + ); + } + + return $sPath; + } + + // -------------------------------------------------------------------------- + + /** + * Re-reads the job so callers always see live state + * + * @throws FactoryException + * @throws ModelException + */ + protected function refresh(Resource\User\Import $oImport): Resource\User\Import + { + $oModel = $this->getModel(); + + /** @var Resource\User\Import|null $oRefreshed */ + $oRefreshed = $oModel->getById($oImport->id); + + return $oRefreshed ?? $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * The system keys which mark the log as belonging to a user import + * + * The UserImport key is what stops the CDN's unused-object monitor reaping + * the log; see Cdn\Monitor\User\ImportCsv. Broken out as its own method + * because it is the one part of storing the log which needs module-cdn's + * MetaData interfaces, and so the one part this module cannot exercise on + * its own. + * + * @return array + */ + protected function getLogMetaData(Resource\User\Import $oImport): array + { + return [ + [ + 'key' => (new SystemKey\UserImport())->get(), + 'value' => true, + ], + [ + 'key' => (new SystemKey\ImportedFrom())->get(), + 'value' => $oImport->object_id, + ], + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the logger + * + * Broken out so that what a job reports can be asserted without writing to + * the filesystem, as Model\User\Import::getDb() does for the database. + * + * @throws FactoryException + */ + protected function getLogger(): Logger + { + /** @var Logger $oLogger */ + $oLogger = Factory::service('Logger'); + return $oLogger; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the CDN service + * + * Broken out so that the log-attach contract can be exercised against each + * of objectCreate()'s three outcomes without standing up a CDN. + * + * @throws FactoryException + */ + protected function getCdn(): Cdn\Service\Cdn + { + /** @var Cdn\Service\Cdn $oCdn */ + $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + return $oCdn; + } + + // -------------------------------------------------------------------------- + + /** + * Hands the finished job to the app, keeping hold of the reason if it objects + * + * Deliberately unable to change the job's status: every account which was + * going to be created exists by the time this runs, so a hook which throws + * is a note against a finished import, not a failed one. The reason is + * appended to `error` rather than replacing it - complete() may already have + * recorded that the log could not be attached, and both matter. + * + * @throws FactoryException + * @throws ModelException + */ + protected function fireImportComplete(Resource\User\Import $oImport): Resource\User\Import + { + try { + $this->getImportService()->onImportComplete($oImport); + + return $oImport; + + } catch (Throwable $e) { + + $this->report($oImport, 'onImportComplete() failed', $e); + + $sError = $this->truncateError(trim(sprintf( + "%s\n\n%s", + (string) $oImport->error, + sprintf( + 'The import itself finished normally, but the application ' + . 'reported a problem afterwards: %s', + $e->getMessage() + ) + ))); + + if (!$this->getModel()->update($oImport->id, ['error' => $sError])) { + $this->report($oImport, sprintf( + 'failed to record the onImportComplete() failure; %s', + $this->getModel()->lastError() ?: 'no reason was reported' + )); + } + + return $this->refresh($oImport); + } + } + + // -------------------------------------------------------------------------- + + /** + * Tells the app the job was rejected + * + * Reported and otherwise swallowed. Nothing may escape: fail() is reached + * from process()'s catch-all, so a throw from here would recurse straight + * back into it - the same reason fail()'s own update is checked rather than + * asserted. The reason is deliberately not written to the job either; that + * column holds the composed explanation of why the import was rejected, and + * this is not it. + */ + protected function fireImportFailed(Resource\User\Import $oImport): void + { + try { + $this->getImportService()->onImportFailed($oImport); + + } catch (Throwable $e) { + $this->report($oImport, 'onImportFailed() failed', $e); + } + } + + // -------------------------------------------------------------------------- + + /** + * Returns the user model + * + * Broken out alongside getImportService(): the per-row loop is where both + * row hooks fire, and what they do to a row cannot be asserted without + * standing an account creation up otherwise. + * + * @throws FactoryException + */ + protected function getUserModel(): Model\User + { + /** @var Model\User $oUserModel */ + $oUserModel = Factory::model('User', Constants::MODULE_SLUG); + return $oUserModel; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the import service, which is where an app hooks into a job + * + * Broken out for the same reason Validator::getImportService() is: the app + * subclasses it, so what a hook does to a job has to be assertable without + * standing one up. + * + * @throws FactoryException + */ + protected function getImportService(): ImportService + { + /** @var ImportService $oImportService */ + $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); + return $oImportService; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the job model + * + * @throws FactoryException + */ + protected function getModel(): Model\User\Import + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + return $oModel; + } +} diff --git a/src/Service/User/Import/Validator.php b/src/Service/User/Import/Validator.php new file mode 100644 index 00000000..3bf5e418 --- /dev/null +++ b/src/Service/User/Import/Validator.php @@ -0,0 +1,394 @@ +getImportService(); + + $aKeys = $oImportService->getKeys(); + $aUniqueKeys = $oImportService->getUniqueKeys(); + + foreach ($this->getIdentityKeys() as $sKey) { + + if (!in_array($sKey, $aKeys, true)) { + throw new TemplateException(sprintf( + 'The user import template cannot be used: accounts are identified by "%s" ' + . '(APP_NATIVE_LOGIN_USING is "%s"), but it is not among the columns returned ' + . 'by %s::getKeys().', + $sKey, + (string) Config::get('APP_NATIVE_LOGIN_USING'), + $oImportService::class + )); + } + + if (!in_array($sKey, $aUniqueKeys, true)) { + throw new TemplateException(sprintf( + 'The user import template cannot be used: accounts are identified by "%s" ' + . '(APP_NATIVE_LOGIN_USING is "%s"), so it must be among the columns returned ' + . 'by %s::getUniqueKeys().', + $sKey, + (string) Config::get('APP_NATIVE_LOGIN_USING'), + $oImportService::class + )); + } + } + } + + // -------------------------------------------------------------------------- + + /** + * The keys an account cannot be created without + * + * Mirrors the branches of Model\User::create(); note that BOTH means both are + * needed, not either. + * + * @return string[] + */ + protected function getIdentityKeys(): array + { + return match (Config::get('APP_NATIVE_LOGIN_USING')) { + 'EMAIL' => ['email'], + 'USERNAME' => ['username'], + default => ['email', 'username'], + }; + } + + // -------------------------------------------------------------------------- + + /** + * Validates the CSV's header row + * + * @param string[] $aHeader The header row + * + * @throws ValidationException + * @throws FactoryException + */ + public function validateHeader(array $aHeader): void + { + $oImportService = $this->getImportService(); + + if (empty($aHeader)) { + throw new ValidationException( + 'Missing header row' + ); + } + + $aDiff = array_diff($aHeader, $oImportService->getKeys()); + if (!empty($aDiff)) { + throw new ValidationException(sprintf( + 'Header row contains the following invalid values: %s', + implode(', ', $aDiff) + )); + } + + /** + * The diff above only catches columns we do not recognise; a CSV may + * legitimately omit most of them, and the per-field rules only apply to + * the columns actually supplied. The identity columns are the exception: + * without them Model\User::create() cannot make an account, so a file + * which omits one used to pass here and then fail on every single row. + */ + $aMissing = array_diff($this->getIdentityKeys(), $aHeader); + if (!empty($aMissing)) { + throw new ValidationException(sprintf( + 'Header row is missing the following %s, without which an account cannot be identified: %s', + count($aMissing) === 1 ? 'column' : 'columns', + implode(', ', $aMissing) + )); + } + } + + // -------------------------------------------------------------------------- + + /** + * Validates a set of raw data rows against the import service's rules + * + * @param string[] $aHeader The CSV's header row + * @param iterable $aRows Raw rows, keyed by line number + * + * @return array The errors, keyed by line number + * @throws FactoryException + * @throws NailsException + */ + public function validateRows(array $aHeader, iterable $aRows): array + { + $oImportService = $this->getImportService(); + /** @var FormValidation $oFormValidationService */ + $oFormValidationService = Factory::service('FormValidation'); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + + /** + * Key the rules by the header's own columns; the CSV need not use every + * key, nor list them in the same order as getKeys() + */ + $aValidationRules = array_filter( + array_map( + fn($sKey) => $oImportService->getValidationRules($sKey), + array_combine($aHeader, $aHeader) + ) + ); + + /** + * One validator, reused per row; each run() replaces the previous + * result so errors never leak between rows. + */ + $oRowValidator = $oFormValidationService->buildValidator( + aRules: $aValidationRules, + aData: [] + ); + + $aErrors = []; + + foreach ($aRows as $iLine => $aRow) { + + $aLineErrors = []; + + if (count($aRow) !== count($aHeader)) { + $aLineErrors[] = sprintf( + 'Row has %d columns, the header has %d', + count($aRow), + count($aHeader) + ); + } + + try { + + $oRowValidator->run($oCsv->align($aHeader, $aRow)); + + } catch (ValidationException $e) { + + foreach ($e->getData() as $sKey => $sError) { + $aLineErrors[] = sprintf('%s: %s', $sKey, $sError); + } + + } catch (Throwable $e) { + $aLineErrors[] = $e->getMessage(); + } + + if (!empty($aLineErrors)) { + $aErrors[$iLine] = $aLineErrors; + } + } + + return $aErrors; + } + + // -------------------------------------------------------------------------- + + /** + * Detects values which are duplicated within the CSV itself + * + * Duplicate detection cannot live in the per-field rules; those only ever + * see a single value, and the validator abandons a field's remaining rules + * as soon as one of them fails. It's a whole-of-file concern, so handle it + * as one streaming pass over the file, holding only the values seen so far + * for the unique keys. + * + * @param string $sPath The path to the CSV + * + * @return string[] + * @throws FactoryException + */ + public function detectDuplicates(string $sPath): array + { + $oImportService = $this->getImportService(); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + + $aHeader = $oCsv->getHeader($sPath); + $aErrors = []; + $aSeen = []; + $aColumn = []; + + foreach ($oImportService->getUniqueKeys() as $sKey) { + + $iColumn = array_search($sKey, $aHeader, true); + if ($iColumn === false) { + continue; + } + + $aColumn[$sKey] = $iColumn; + $aSeen[$sKey] = []; + } + + if (empty($aColumn)) { + return []; + } + + foreach ($oCsv->readRawRows($sPath) as $iLine => $aRow) { + foreach ($aColumn as $sKey => $iColumn) { + + $sValue = strtolower(trim((string) ($aRow[$iColumn] ?? ''))); + if ($sValue === '') { + continue; + } + + if (array_key_exists($sValue, $aSeen[$sKey])) { + $aErrors[] = sprintf( + 'Line %d: %s: "%s" must only appear once; it is also on line %d', + $iLine, + $sKey, + $sValue, + $aSeen[$sKey][$sValue] + ); + } else { + $aSeen[$sKey][$sValue] = $iLine; + } + } + } + + return $aErrors; + } + + // -------------------------------------------------------------------------- + + /** + * Detects values which are already registered against an existing account + * + * Batched, not per row: one streaming pass collects the unique keys' values, + * then one lookup per key answers for all of them. The closures this + * replaced ran a query - two, on a hit - for every line in the file. + * + * Takes rows rather than a path so that the upload can hand it the whole + * file as a generator while the runner hands it a chunk. + * + * Note that this is an early warning, not the authority: Model\User::create() + * performs its own uniqueness check, so a value registered after this ran is + * still refused at the point of creation. + * + * @param string[] $aHeader The CSV's header row + * @param iterable $aRows Raw rows, keyed by line number + * + * @return array The errors, keyed by line number + * @throws FactoryException + * @throws TemplateException + */ + public function detectRegistered(array $aHeader, iterable $aRows): array + { + $oImportService = $this->getImportService(); + + $aColumn = []; + $aLines = []; + + foreach ($oImportService->getUniqueKeys() as $sKey) { + + $iColumn = array_search($sKey, $aHeader, true); + if ($iColumn === false) { + continue; + } + + $aColumn[$sKey] = $iColumn; + $aLines[$sKey] = []; + } + + if (empty($aColumn)) { + return []; + } + + /** + * Only the values are held, not the rows - and keyed by value, because a + * value can legitimately recur within a chunk even though the upload's + * duplicate check would reject it across the whole file. + */ + foreach ($aRows as $iLine => $aRow) { + foreach ($aColumn as $sKey => $iColumn) { + + $sValue = strtolower(trim((string) ($aRow[$iColumn] ?? ''))); + if ($sValue === '') { + continue; + } + + $aLines[$sKey][$sValue][] = $iLine; + } + } + + $aErrors = []; + + foreach ($aLines as $sKey => $aValues) { + foreach ($oImportService->whichExist($sKey, array_keys($aValues)) as $sValue) { + foreach ($aValues[$sValue] ?? [] as $iLine) { + $aErrors[$iLine][] = sprintf('%s: "%s" is already registered', $sKey, $sValue); + } + } + } + + ksort($aErrors); + + return $aErrors; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the import service + * + * Broken out so the template and unique-key contracts can be exercised + * against an app which has amended them, without registering a service; the + * same seam Model\User\Import::getDb() provides for the database. + * + * @throws FactoryException + */ + protected function getImportService(): ImportService + { + /** @var ImportService $oImportService */ + $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); + return $oImportService; + } +} diff --git a/tests/Enum/User/Import/StatusTest.php b/tests/Enum/User/Import/StatusTest.php new file mode 100644 index 00000000..d7a5b155 --- /dev/null +++ b/tests/Enum/User/Import/StatusTest.php @@ -0,0 +1,87 @@ +isTerminal()); + self::assertTrue(Status::PARTIAL->isTerminal()); + self::assertTrue(Status::FAILED->isTerminal()); + } + + // -------------------------------------------------------------------------- + + public function test_unfinished_statuses_are_not_terminal(): void + { + self::assertFalse(Status::DRAFT->isTerminal()); + self::assertFalse(Status::PENDING->isTerminal()); + self::assertFalse(Status::VALIDATING->isTerminal()); + self::assertFalse(Status::RUNNING->isTerminal()); + } + + // -------------------------------------------------------------------------- + + /** + * A draft has not been approved, so no runner may touch it + */ + public function test_a_draft_is_not_active(): void + { + self::assertFalse(Status::DRAFT->isActive()); + self::assertTrue(Status::PENDING->isActive()); + self::assertTrue(Status::VALIDATING->isActive()); + self::assertTrue(Status::RUNNING->isActive()); + } + + // -------------------------------------------------------------------------- + + /** + * Only a job a runner holds mid-flight is protected; PENDING is approved but + * untouched, so it is deletable even though it is active + */ + public function test_only_in_flight_statuses_are_not_deletable(): void + { + self::assertFalse(Status::VALIDATING->isDeletable()); + self::assertFalse(Status::RUNNING->isDeletable()); + + self::assertTrue(Status::DRAFT->isDeletable()); + self::assertTrue(Status::PENDING->isDeletable()); + self::assertTrue(Status::COMPLETE->isDeletable()); + self::assertTrue(Status::PARTIAL->isDeletable()); + self::assertTrue(Status::FAILED->isDeletable()); + } + + // -------------------------------------------------------------------------- + + /** + * Guards against `isActive()` being mistaken for the deletion rule; it is + * not, because it includes PENDING + */ + public function test_deletability_is_not_the_inverse_of_active(): void + { + self::assertTrue(Status::PENDING->isActive()); + self::assertTrue(Status::PENDING->isDeletable()); + } + + // -------------------------------------------------------------------------- + + public function test_values_maps_onto_the_database_representation(): void + { + self::assertSame( + ['PENDING', 'VALIDATING', 'RUNNING'], + Status::values(Status::active()) + ); + + self::assertSame( + ['COMPLETE', 'PARTIAL', 'FAILED'], + Status::values(Status::terminal()) + ); + } +} diff --git a/tests/Model/User/ImportModelTest.php b/tests/Model/User/ImportModelTest.php new file mode 100644 index 00000000..65ca549e --- /dev/null +++ b/tests/Model/User/ImportModelTest.php @@ -0,0 +1,229 @@ +getProperty('CACHING_ENABLED') + ->getValue() + ); + } + + // -------------------------------------------------------------------------- + + public function test_the_expected_relationships_are_registered(): void + { + $aFields = []; + foreach ((new Import())->getExpandableFields() as $oField) { + $aFields[$oField->trigger] = $oField; + } + + self::assertArrayHasKey('user', $aFields); + self::assertSame('created_by', $aFields['user']->id_column); + self::assertSame(Constants::MODULE_SLUG, $aFields['user']->provider); + + self::assertArrayHasKey('object', $aFields); + self::assertSame('object_id', $aFields['object']->id_column); + + self::assertArrayHasKey('log', $aFields); + self::assertSame('log_id', $aFields['log']->id_column); + } + + // -------------------------------------------------------------------------- + + public function test_claiming_guards_against_a_job_which_is_already_held(): void + { + $oSpy = new DatabaseSpy(); + $oModel = new ImportModelWithSpy($oSpy); + + $oModel->claim(7, 'token-a'); + + // The update only lands if nobody else holds the job... + self::assertContains(['claim_token', null], $oSpy->callsTo('where')); + + // ...and only for a status a runner is allowed to pick up + self::assertContains( + ['status', ['PENDING', 'VALIDATING', 'RUNNING']], + $oSpy->callsTo('where_in') + ); + + self::assertSame( + Status::values(Status::active()), + ['PENDING', 'VALIDATING', 'RUNNING'] + ); + + self::assertContains(['claim_token', 'token-a'], $oSpy->callsTo('set')); + self::assertContains('update', $oSpy->methods()); + } + + // -------------------------------------------------------------------------- + + public function test_a_claim_is_confirmed_by_reading_the_row_back(): void + { + $oSpy = new DatabaseSpy(); + $oSpy->iCountAllResults = 1; + + self::assertTrue((new ImportModelWithSpy($oSpy))->claim(7, 'token-a')); + + // The read back must be scoped to this process's own token + self::assertContains(['id', 7], $oSpy->callsTo('where')); + self::assertContains(['claim_token', 'token-a'], $oSpy->callsTo('where')); + } + + // -------------------------------------------------------------------------- + + public function test_losing_the_race_yields_no_claim(): void + { + $oSpy = new DatabaseSpy(); + $oSpy->iCountAllResults = 0; + + self::assertFalse((new ImportModelWithSpy($oSpy))->claim(7, 'token-a')); + } + + // -------------------------------------------------------------------------- + + public function test_releasing_clears_the_claim(): void + { + $oSpy = new DatabaseSpy(); + + (new ImportModelWithSpy($oSpy))->release(7); + + self::assertContains(['claim_token', null], $oSpy->callsTo('set')); + self::assertContains(['claimed', null], $oSpy->callsTo('set')); + self::assertContains(['id', 7], $oSpy->callsTo('where')); + self::assertContains('update', $oSpy->methods()); + } + + // -------------------------------------------------------------------------- + + /** + * A restarted job has to lose its items: the unique key on + * (import_id, line) would otherwise make every line look already handled. + */ + public function test_deleting_a_jobs_items_targets_only_that_job(): void + { + $oSpy = new DatabaseSpy(); + + self::assertTrue((new ImportModelWithSpy($oSpy))->deleteItems(7)); + + self::assertContains(['import_id', 7], $oSpy->callsTo('where')); + self::assertContains('delete', $oSpy->methods()); + + // The items table, not the jobs table + self::assertSame( + [Import\Item::TABLE], + array_map(fn(array $aArgs): string => $aArgs[0], $oSpy->callsTo('delete')) + ); + } + + // -------------------------------------------------------------------------- + + /** + * Builds a job resource from the columns the model would supply + */ + private function makeImport(?int $iObjectId, ?int $iLogId): Resource\User\Import + { + return new Resource\User\Import((object) [ + 'id' => 7, + 'object_id' => $iObjectId, + 'log_id' => $iLogId, + 'additional' => '{}', + 'status' => 'COMPLETE', + 'runner' => 'CRON', + 'claim_token' => null, + 'claimed' => null, + 'error' => null, + 'row_count' => 10, + 'validated_count' => 10, + 'processed_count' => 10, + 'success_count' => 10, + 'error_count' => 0, + 'started' => null, + 'finished' => null, + ]); + } + + // -------------------------------------------------------------------------- + + public function test_only_the_csv_is_destroyed_when_there_is_no_log(): void + { + $oCdn = new CdnSpy(); + + $aErrors = (new ImportModelWithSpy(new DatabaseSpy(), $oCdn)) + ->destroyObjects($this->makeImport(2, null)); + + self::assertSame([2], $oCdn->aDestroyed); + self::assertSame([], $aErrors); + } + + // -------------------------------------------------------------------------- + + public function test_both_the_csv_and_the_log_are_destroyed(): void + { + $oCdn = new CdnSpy(); + + $aErrors = (new ImportModelWithSpy(new DatabaseSpy(), $oCdn)) + ->destroyObjects($this->makeImport(2, 3)); + + self::assertSame([2, 3], $oCdn->aDestroyed); + self::assertSame([], $aErrors); + } + + // -------------------------------------------------------------------------- + + /** + * objectDestroy() reports a missing object by returning false, not by + * throwing, so the return value has to be checked + */ + public function test_a_false_return_is_collected_with_its_error(): void + { + $oCdn = new CdnSpy(); + $oCdn->aFailures = [3 => 'Nothing to destroy.']; + + $aErrors = (new ImportModelWithSpy(new DatabaseSpy(), $oCdn)) + ->destroyObjects($this->makeImport(2, 3)); + + self::assertSame([3 => 'Nothing to destroy.'], $aErrors); + } + + // -------------------------------------------------------------------------- + + public function test_a_throw_is_collected_with_its_message(): void + { + $oCdn = new CdnSpy(); + $oCdn->aThrows = [2 => 'The driver fell over']; + + $aErrors = (new ImportModelWithSpy(new DatabaseSpy(), $oCdn)) + ->destroyObjects($this->makeImport(2, 3)); + + // The log is still attempted; one bad object must not strand the other + self::assertSame([2, 3], $oCdn->aDestroyed); + self::assertSame([2 => 'The driver fell over'], $aErrors); + } +} diff --git a/tests/Resource/User/ImportTest.php b/tests/Resource/User/ImportTest.php new file mode 100644 index 00000000..3e2e457b --- /dev/null +++ b/tests/Resource/User/ImportTest.php @@ -0,0 +1,134 @@ + 1, + 'object_id' => 2, + 'log_id' => null, + 'additional' => '{}', + 'status' => 'RUNNING', + 'runner' => 'CRON', + 'claim_token' => null, + 'claimed' => null, + 'error' => null, + 'row_count' => 100, + 'validated_count' => 100, + 'processed_count' => 25, + 'success_count' => 25, + 'error_count' => 0, + 'started' => null, + 'finished' => null, + ], $aOverrides)); + } + + // -------------------------------------------------------------------------- + + /** + * Both columns arrived with the feature rather than with the table, so the + * resource has to tolerate a schema which has not caught up - the `make()` + * above deliberately omits them for that reason. + */ + public function test_the_columns_added_after_the_table_tolerate_a_lagging_schema(): void + { + $oImport = $this->make(); + + self::assertFalse($oImport->skip_registered); + self::assertSame(0, $oImport->warning_count); + } + + // -------------------------------------------------------------------------- + + public function test_a_warning_count_is_read_when_it_is_there(): void + { + self::assertSame(3, $this->make(['warning_count' => '3'])->warning_count); + } + + // -------------------------------------------------------------------------- + + public function test_the_status_is_cast_to_an_enum(): void + { + self::assertSame(Status::RUNNING, $this->make()->status); + } + + // -------------------------------------------------------------------------- + + public function test_the_runner_is_cast_to_an_enum(): void + { + self::assertSame(Runner::CRON, $this->make()->runner); + self::assertNull($this->make(['runner' => null])->runner); + } + + // -------------------------------------------------------------------------- + + public function test_the_additional_fields_are_decoded(): void + { + $oResource = $this->make(['additional' => '{"source":"crm"}']); + + self::assertInstanceOf(stdClass::class, $oResource->additional); + self::assertSame('crm', $oResource->additional->source); + } + + // -------------------------------------------------------------------------- + + public function test_missing_additional_fields_decode_to_an_empty_object(): void + { + self::assertEquals(new stdClass(), $this->make(['additional' => null])->additional); + } + + // -------------------------------------------------------------------------- + + public function test_progress_is_measured_against_the_running_cursor(): void + { + self::assertSame(25, $this->make()->getPercent()); + } + + // -------------------------------------------------------------------------- + + public function test_progress_is_measured_against_the_validating_cursor(): void + { + $oResource = $this->make([ + 'status' => 'VALIDATING', + 'validated_count' => 40, + 'processed_count' => 0, + ]); + + self::assertSame(40, $oResource->getPercent()); + } + + // -------------------------------------------------------------------------- + + public function test_a_finished_job_is_always_complete(): void + { + foreach (Status::terminal() as $oStatus) { + self::assertSame( + 100, + $this->make(['status' => $oStatus->value, 'processed_count' => 3])->getPercent(), + $oStatus->value + ); + } + } + + // -------------------------------------------------------------------------- + + public function test_a_job_of_unknown_size_reports_no_progress(): void + { + self::assertSame(0, $this->make(['row_count' => null])->getPercent()); + } +} diff --git a/tests/Service/User/Import/CsvTest.php b/tests/Service/User/Import/CsvTest.php new file mode 100644 index 00000000..292f62ec --- /dev/null +++ b/tests/Service/User/Import/CsvTest.php @@ -0,0 +1,278 @@ +oCsv = $oCsv; + } + + // -------------------------------------------------------------------------- + + protected function tearDown(): void + { + foreach ($this->aPaths as $sPath) { + @unlink($sPath); + } + $this->aPaths = []; + } + + // -------------------------------------------------------------------------- + + /** + * Writes a CSV to a temporary file and returns its path + */ + private function write(string $sContents): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-user-import-test-'); + file_put_contents($sPath, $sContents); + $this->aPaths[] = $sPath; + return $sPath; + } + + // -------------------------------------------------------------------------- + + public function test_get_header_returns_the_first_record(): void + { + $sPath = $this->write("email,first_name,last_name\na@example.com,Ada,Lovelace\n"); + + self::assertSame( + ['email', 'first_name', 'last_name'], + $this->oCsv->getHeader($sPath) + ); + } + + // -------------------------------------------------------------------------- + + public function test_get_header_trims_whitespace(): void + { + $sPath = $this->write("email , first_name\na@example.com,Ada\n"); + + self::assertSame( + ['email', 'first_name'], + $this->oCsv->getHeader($sPath) + ); + } + + // -------------------------------------------------------------------------- + + public function test_get_header_of_an_empty_file_is_empty(): void + { + self::assertSame([], $this->oCsv->getHeader($this->write(''))); + } + + // -------------------------------------------------------------------------- + + public function test_count_rows_excludes_the_header(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\nc@example.com\n"); + + self::assertSame(3, $this->oCsv->countRows($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_count_rows_ignores_blank_lines(): void + { + $sPath = $this->write("email\na@example.com\n\nb@example.com\n\n"); + + self::assertSame(2, $this->oCsv->countRows($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_read_raw_rows_keys_by_line_number(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\n"); + + self::assertSame( + [ + 2 => ['a@example.com'], + 3 => ['b@example.com'], + ], + iterator_to_array($this->oCsv->readRawRows($sPath), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_read_raw_rows_honours_the_offset_and_limit(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\nc@example.com\nd@example.com\n"); + + self::assertSame( + [ + 3 => ['b@example.com'], + 4 => ['c@example.com'], + ], + iterator_to_array($this->oCsv->readRawRows($sPath, 1, 2), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_read_raw_rows_past_the_end_is_empty(): void + { + $sPath = $this->write("email\na@example.com\n"); + + self::assertSame( + [], + iterator_to_array($this->oCsv->readRawRows($sPath, 10, 5), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_read_raw_rows_respects_quoted_separators(): void + { + $sPath = $this->write("email,last_name\n\"a@example.com\",\"Lovelace, Ada\"\n"); + + self::assertSame( + [2 => ['a@example.com', 'Lovelace, Ada']], + iterator_to_array($this->oCsv->readRawRows($sPath), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_align_keys_the_row_by_the_header(): void + { + self::assertSame( + ['email' => 'a@example.com', 'first_name' => 'Ada'], + $this->oCsv->align(['email', 'first_name'], ['a@example.com', 'Ada']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_align_pads_short_rows(): void + { + self::assertSame( + ['email' => 'a@example.com', 'first_name' => ''], + $this->oCsv->align(['email', 'first_name'], ['a@example.com']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_align_truncates_long_rows(): void + { + self::assertSame( + ['email' => 'a@example.com'], + $this->oCsv->align(['email'], ['a@example.com', 'Ada']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_align_trims_values(): void + { + self::assertSame( + ['email' => 'a@example.com'], + $this->oCsv->align(['email'], [' a@example.com ']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_apply_defaults_replaces_blank_cells(): void + { + // first_name has no default, so a blank cell becomes null rather than '' + self::assertSame( + ['email' => 'a@example.com', 'first_name' => null], + $this->oCsv->applyDefaults(['email' => 'a@example.com', 'first_name' => '']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_apply_defaults_leaves_populated_cells_alone(): void + { + self::assertSame( + ['first_name' => 'Ada'], + $this->oCsv->applyDefaults(['first_name' => 'Ada']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_read_rows_combines_and_defaults(): void + { + $sPath = $this->write("email,first_name\na@example.com,Ada\nb@example.com,\n"); + + self::assertSame( + [ + 2 => ['email' => 'a@example.com', 'first_name' => 'Ada'], + 3 => ['email' => 'b@example.com', 'first_name' => null], + ], + iterator_to_array($this->oCsv->readRows($sPath), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_write_log_appends_the_outcome_columns(): void + { + $sPath = $this->write("email,first_name\na@example.com,Ada\nb@example.com,Grace\n"); + + $sLog = $this->oCsv->writeLog($sPath, [ + 2 => ['id' => 10, 'status' => 'SUCCESS', 'message' => ''], + 3 => ['id' => null, 'status' => 'ERROR', 'message' => 'Nope'], + ]); + + $this->aPaths[] = $sLog; + + self::assertSame( + implode("\n", [ + 'email,first_name,id,status,message', + 'a@example.com,Ada,10,SUCCESS,', + 'b@example.com,Grace,,ERROR,Nope', + '', + ]), + file_get_contents($sLog) + ); + } + + // -------------------------------------------------------------------------- + + public function test_write_log_only_emits_the_supplied_lines(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\nc@example.com\n"); + + $sLog = $this->oCsv->writeLog($sPath, [ + 3 => ['id' => null, 'status' => 'ERROR', 'message' => 'Nope'], + ]); + + $this->aPaths[] = $sLog; + + self::assertSame( + implode("\n", [ + 'email,id,status,message', + 'b@example.com,,ERROR,Nope', + '', + ]), + file_get_contents($sLog) + ); + } +} diff --git a/tests/Service/User/Import/ProcessorTest.php b/tests/Service/User/Import/ProcessorTest.php new file mode 100644 index 00000000..8e26377b --- /dev/null +++ b/tests/Service/User/Import/ProcessorTest.php @@ -0,0 +1,1346 @@ +oModel = new ImportModelRecorder(); + $this->oCdn = new CdnSpy(); + $this->oLogger = new LoggerSpy(); + $this->oProcessor = new ProcessorWithSpies($this->oModel, $this->oCdn, $this->oLogger); + } + + // -------------------------------------------------------------------------- + + protected function tearDown(): void + { + foreach ($this->aPaths as $sPath) { + if (is_file($sPath)) { + unlink($sPath); + } + } + + $this->aPaths = []; + } + + // -------------------------------------------------------------------------- + + /** + * Builds a job from the columns the model would supply + * + * created_by is left empty on purpose: notify() returns early without one, + * which keeps the email machinery out of these tests. + */ + private function make(array $aOverrides = []): Resource\User\Import + { + return new Resource\User\Import((object) array_merge([ + 'id' => 6, + 'object_id' => 20102, + 'log_id' => null, + 'additional' => '{}', + 'status' => 'VALIDATING', + 'runner' => 'CRON', + 'claim_token' => null, + 'claimed' => null, + 'error' => null, + 'row_count' => 1362, + 'validated_count' => 1362, + 'processed_count' => 0, + 'success_count' => 0, + 'error_count' => 2, + 'started' => null, + 'finished' => null, + 'created_by' => null, + ], $aOverrides)); + } + + // -------------------------------------------------------------------------- + + /** + * Writes a CSV for the log builder to read the source rows back out of + */ + private function makeCsv(int $iRows = 3): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-processor-test-'); + $this->aPaths[] = $sPath; + + $aLines = ['email,first_name,last_name']; + for ($i = 1; $i <= $iRows; $i++) { + $aLines[] = sprintf('user%d@example.com,First%d,Last%d', $i, $i, $i); + } + + file_put_contents($sPath, implode("\n", $aLines) . "\n"); + + return $sPath; + } + + // -------------------------------------------------------------------------- + + /** + * The two rows import #6 actually failed on + */ + private function importSixSample(): array + { + return [ + ['line' => 2, 'message' => 'email: "a.ahmad6@nhs.net" is already registered'], + ['line' => 3, 'message' => 'email: "a.botros@nhs.net" is already registered'], + ]; + } + + // -------------------------------------------------------------------------- + + /** + * A row whose account exists but whose after-create work did not happen + */ + private function importSixWarningSample(): array + { + return [ + ['line' => 4, 'message' => 'The account was created, but the subscription could not be started'], + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Writes a CSV which the real validator will pass + * + * The header carries every column an account can be identified by, whichever + * APP_NATIVE_LOGIN_USING is configured, plus the one other column the + * template requires; without all three, validate() rejects the file before + * it ever reaches onImportStart(). + */ + private function makeValidCsv(int $iRows = 1): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-processor-test-'); + $this->aPaths[] = $sPath; + + $aLines = ['email,username,send_email']; + for ($i = 1; $i <= $iRows; $i++) { + $aLines[] = sprintf('user%d@example.com,user_%d,0', $i, $i); + } + + file_put_contents($sPath, implode("\n", $aLines) . "\n"); + + return $sPath; + } + + // -------------------------------------------------------------------------- + // The seams + // -------------------------------------------------------------------------- + + /** + * The seams are trivial, but a mistake in one is invisible to every other + * test in here - they all override them - so they are exercised against the + * real class. + */ + public function test_the_seams_hand_back_the_real_services(): void + { + $oProcessor = new Processor(); + $oClass = new ReflectionClass($oProcessor); + + /** + * getCdn() is left out: the real CDN service wants a cache directory + * this module does not have on its own, which is the same reason CdnSpy + * replaces its constructor. + */ + $aSeams = [ + 'getModel' => ImportModel::class, + 'getLogger' => Logger::class, + ]; + + foreach ($aSeams as $sMethod => $sExpected) { + self::assertInstanceOf( + $sExpected, + $oClass->getMethod($sMethod)->invoke($oProcessor), + $sMethod . '() should return a ' . $sExpected + ); + } + } + + // -------------------------------------------------------------------------- + // Storing the log + // -------------------------------------------------------------------------- + + /** + * The regression test for import #6. + * + * Cdn::objectCreate() hands back a plain stdClass, not a + * Cdn\Resource\CdnObject; uploadLog() used to declare the resource as its + * return type, so the TypeError on the way out was swallowed by fail()'s + * empty catch and the log was silently never attached. + */ + public function test_the_shape_the_cdn_actually_returns_is_accepted(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = (object) ['id' => '20103']; + + self::assertSame( + 20103, + $this->oProcessor->exposeUploadLog($this->make(), []) + ); + } + + // -------------------------------------------------------------------------- + + public function test_the_log_is_stored_in_the_import_bucket_as_a_csv(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + + $this->oProcessor->exposeUploadLog($this->make(), []); + + self::assertCount(1, $this->oCdn->aCreated); + + $aCreated = $this->oCdn->aCreated[0]; + self::assertSame(Processor::IMPORT_BUCKET, $aCreated['bucket']); + self::assertSame('text/csv', $aCreated['options']['Content-Type']); + self::assertTrue($aCreated['options']['no-md5-check']); + self::assertStringEndsWith('.csv', $aCreated['options']['filename_display']); + } + + // -------------------------------------------------------------------------- + + public function test_a_cdn_which_cannot_store_the_log_reports_its_reason(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'The file is too large, maximum file size is 1 B'; + + $this->expectException(LogException::class); + $this->expectExceptionMessage('The file is too large, maximum file size is 1 B'); + + $this->oProcessor->exposeUploadLog($this->make(), []); + } + + // -------------------------------------------------------------------------- + + /** + * lastError() returns false, not '', for an empty error stack - so without a + * fallback a silent CDN failure would produce a message which trails off. + */ + public function test_a_cdn_failure_with_no_reason_still_says_something(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = false; + + $this->expectException(LogException::class); + $this->expectExceptionMessage('no reason was reported'); + + $this->oProcessor->exposeUploadLog($this->make(), []); + } + + // -------------------------------------------------------------------------- + + public function test_a_log_which_cannot_be_built_is_reported_as_such(): void + { + // Never written, so the log builder cannot read the source rows + $this->oProcessor->sSourcePath = '/nonexistent/user-import.csv'; + + $this->expectException(LogException::class); + $this->expectExceptionMessage('Failed to build the import log'); + + $this->oProcessor->exposeUploadLog($this->make(), []); + } + + // -------------------------------------------------------------------------- + + public function test_a_log_which_cannot_be_attached_is_kept_hold_of_rather_than_swallowed(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'Failed to create object on storage service'; + + $aLog = $this->oProcessor->exposeAttachLog($this->make(['log_id' => 999]), [ItemStatus::ERROR]); + + // The reason comes back to the caller... + self::assertStringContainsString('Failed to create object on storage service', $aLog['error']); + + // ...it is written to the application log... + self::assertCount(1, $this->oLogger->aErrors); + self::assertStringContainsString('User import #6', $this->oLogger->aErrors[0]); + self::assertStringContainsString('failed to attach the log', $this->oLogger->aErrors[0]); + + // ...and an existing log is not thrown away in the process + self::assertSame(999, $aLog['log_id']); + } + + // -------------------------------------------------------------------------- + // fail() + // -------------------------------------------------------------------------- + + public function test_a_failure_records_the_reason_the_log_and_the_status(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + + $this->oProcessor->exposeFail($this->make(), 'Something went wrong.'); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::FAILED->value, $aUpdate['status']); + self::assertSame('Something went wrong.', $aUpdate['error']); + self::assertSame(20103, $aUpdate['log_id']); + self::assertNotEmpty($aUpdate['finished']); + } + + // -------------------------------------------------------------------------- + + public function test_a_failure_notes_on_itself_when_the_log_could_not_be_attached(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'The CDN is unreachable'; + + $this->oProcessor->exposeFail($this->make(), 'Something went wrong.'); + + $sError = $this->oModel->lastUpdate()['error']; + + self::assertStringContainsString('Something went wrong.', $sError); + self::assertStringContainsString('The error log could not be attached', $sError); + self::assertStringContainsString('The CDN is unreachable', $sError); + } + + // -------------------------------------------------------------------------- + + /** + * A job with nothing wrong per-row has no error log to build, so the CDN + * should not be troubled for one. + */ + public function test_a_failure_with_no_failing_rows_does_not_build_a_log(): void + { + $this->oProcessor->exposeFail($this->make(['error_count' => 0]), 'The CSV had no header.'); + + self::assertSame([], $this->oCdn->aCreated); + self::assertNull($this->oModel->lastUpdate()['log_id']); + } + + // -------------------------------------------------------------------------- + + /** + * Throwing here would re-enter process()'s catch and recurse back into + * fail(), so a failed write is reported and left alone. + */ + public function test_a_failure_which_cannot_be_recorded_is_reported_not_thrown(): void + { + $this->oModel->bUpdateReturn = false; + $this->oModel->sUpdateError = 'Deadlock found when trying to get lock'; + + $this->oProcessor->exposeFail($this->make(['error_count' => 0]), 'Something went wrong.'); + + self::assertStringContainsString( + 'Deadlock found when trying to get lock', + implode("\n", $this->oLogger->aErrors) + ); + } + + // -------------------------------------------------------------------------- + // complete() + // -------------------------------------------------------------------------- + + /** + * complete() used to call the log upload with no try/catch at all, so a log + * failure reached process()'s catch and re-branded a perfectly good import + * as FAILED - after the accounts had been created. + */ + public function test_a_log_failure_does_not_turn_a_finished_import_into_a_failed_one(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'The CDN is unreachable'; + + $oImport = $this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1362, + 'error_count' => 0, + 'log_id' => 555, + ]); + + $this->oProcessor->exposeComplete($oImport); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::COMPLETE->value, $aUpdate['status']); + + // An existing log must not be nulled by a failed upload + self::assertSame(555, $aUpdate['log_id']); + + // ...and the reason has to be visible somewhere + self::assertStringContainsString('The error log could not be attached', $aUpdate['error']); + self::assertStringContainsString('The CDN is unreachable', $aUpdate['error']); + } + + // -------------------------------------------------------------------------- + + public function test_an_import_with_failing_rows_completes_as_partial(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + + $this->oProcessor->exposeComplete($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1360, + 'error_count' => 2, + ])); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::PARTIAL->value, $aUpdate['status']); + self::assertNull($aUpdate['error']); + self::assertSame(20103, $aUpdate['log_id']); + } + + // -------------------------------------------------------------------------- + // begin() + // -------------------------------------------------------------------------- + + /** + * The unique key on (import_id, line) would otherwise make every line of a + * restarted job look as though it had already been handled, and the counts, + * which are re-derived rather than incremented, would come from the run + * before. + */ + public function test_restarting_a_job_clears_the_previous_attempt(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + + $oImport = $this->make([ + 'status' => 'PENDING', + 'log_id' => 20103, + 'validated_count' => 1362, + 'error_count' => 2, + ]); + + $this->oProcessor->exposeBegin($oImport); + + // The previous run's items go... + self::assertSame([6], $this->oModel->aItemsDeleted); + + // ...as does the log which described them... + self::assertSame([20103], $this->oCdn->aDestroyed); + + // ...and the job is reset to the top + $aUpdate = $this->oModel->lastUpdate(); + self::assertNull($aUpdate['log_id']); + self::assertNull($aUpdate['error']); + self::assertSame(0, $aUpdate['validated_count']); + self::assertSame(0, $aUpdate['error_count']); + self::assertSame(Status::VALIDATING->value, $aUpdate['status']); + self::assertSame(3, $aUpdate['row_count']); + } + + // -------------------------------------------------------------------------- + + public function test_a_previous_log_which_cannot_be_destroyed_does_not_stop_the_restart(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->aFailures = [20103 => 'Object does not exist']; + + $this->oProcessor->exposeBegin($this->make(['status' => 'PENDING', 'log_id' => 20103])); + + self::assertSame(Status::VALIDATING->value, $this->oModel->lastUpdate()['status']); + self::assertStringContainsString( + 'Object does not exist', + implode("\n", $this->oLogger->aWarnings) + ); + } + + // -------------------------------------------------------------------------- + // Composing the error + // -------------------------------------------------------------------------- + + public function test_a_validation_failure_names_the_rows_which_failed(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + + $sError = $this->oProcessor->exposeComposeValidationError($this->make()); + + self::assertStringContainsString('2 rows in the CSV could not be validated', $sError); + self::assertStringContainsString('no user accounts were created', $sError); + self::assertStringContainsString('validating the CSV (1,362 of 1,362 rows checked)', $sError); + self::assertStringContainsString('Line 2: email: "a.ahmad6@nhs.net" is already registered', $sError); + self::assertStringContainsString('Line 3: email: "a.botros@nhs.net" is already registered', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_a_single_failing_row_is_worded_in_the_singular(): void + { + $this->oProcessor->aSample = [['line' => 2, 'message' => 'email: is already registered']]; + + self::assertStringContainsString( + '1 row in the CSV could not be validated', + $this->oProcessor->exposeComposeValidationError($this->make(['error_count' => 1])) + ); + } + + // -------------------------------------------------------------------------- + + /** + * The symptom which made import #6's error useless: rows failed, but every + * message was empty, so there was nothing to read either way. + */ + public function test_a_row_with_no_recorded_reason_is_still_listed(): void + { + $this->oProcessor->aSample = [ + ['line' => 2, 'message' => null], + ['line' => 3, 'message' => ''], + ]; + + $sError = $this->oProcessor->exposeComposeValidationError($this->make()); + + self::assertStringContainsString('Line 2: (no reason was recorded)', $sError); + self::assertStringContainsString('Line 3: (no reason was recorded)', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_only_a_sample_of_a_large_failure_is_quoted(): void + { + $aSample = []; + for ($i = 2; $i <= 200; $i++) { + $aSample[] = ['line' => $i, 'message' => 'email: is already registered']; + } + + $this->oProcessor->aSample = $aSample; + + $sError = $this->oProcessor->exposeComposeValidationError($this->make(['error_count' => 199])); + + self::assertStringContainsString( + sprintf('showing the first %s of 199', Processor::ERROR_SAMPLE_SIZE), + $sError + ); + + // One header line plus the sample, and nothing beyond it + self::assertStringContainsString('Line 11: ', $sError); + self::assertStringNotContainsString('Line 12: ', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_an_unexpected_failure_names_the_exception_and_where_it_came_from(): void + { + $e = new RuntimeException('Something broke'); + + $sError = $this->oProcessor->exposeComposeThrowableError($this->make(), $e); + + self::assertStringContainsString('The import stopped unexpectedly', $sError); + self::assertStringContainsString('Cause: RuntimeException - Something broke', $sError); + self::assertStringContainsString('Where: ProcessorTest.php line ' . $e->getLine(), $sError); + self::assertStringContainsString('written to the application log', $sError); + + // A container's absolute paths are noise in an admin's inbox + self::assertStringNotContainsString(dirname($e->getFile()), $sError); + } + + // -------------------------------------------------------------------------- + // summariseError() + // -------------------------------------------------------------------------- + + /** + * The notification is an FYI which lands in an inbox, and the quoted rows + * repeat cell values out of the CSV. The reader is told what happened and + * sent to the log, which is behind a permission check, for the rest. + */ + public function test_the_emailed_error_keeps_the_summary_and_the_phase(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + + $sSummary = $this->oProcessor->exposeSummariseError( + $this->oProcessor->exposeComposeValidationError($this->make(['error_count' => 2])) + ); + + self::assertStringContainsString('could not be validated', $sSummary); + self::assertStringContainsString('Phase: validating the CSV', $sSummary); + } + + // -------------------------------------------------------------------------- + + public function test_the_emailed_error_drops_the_rows_it_quoted(): void + { + $this->oProcessor->aSample = [ + ['line' => 2, 'message' => 'email: "ada@example.com" is already registered'], + ]; + + $sSummary = $this->oProcessor->exposeSummariseError( + $this->oProcessor->exposeComposeValidationError($this->make(['error_count' => 1])) + ); + + self::assertStringNotContainsString('ada@example.com', $sSummary); + self::assertStringNotContainsString('Rows with errors', $sSummary); + self::assertStringNotContainsString('Line 2', $sSummary); + } + + // -------------------------------------------------------------------------- + + /** + * A driver reporting a duplicate key repeats the value which collided, so + * the cause can name somebody even when no row was quoted. + */ + public function test_the_emailed_error_drops_the_cause_and_where_it_came_from(): void + { + $e = new RuntimeException('Duplicate entry \'ada@example.com\' for key \'email\''); + + $sSummary = $this->oProcessor->exposeSummariseError( + $this->oProcessor->exposeComposeThrowableError($this->make(), $e) + ); + + self::assertStringContainsString('The import stopped unexpectedly', $sSummary); + self::assertStringNotContainsString('ada@example.com', $sSummary); + self::assertStringNotContainsString('Cause:', $sSummary); + self::assertStringNotContainsString('Where:', $sSummary); + } + + // -------------------------------------------------------------------------- + + /** + * complete() leaves the error unset on the happy path, and the template + * renders the block only when there is something in it. + */ + public function test_a_job_with_no_error_emails_no_error(): void + { + self::assertNull($this->oProcessor->exposeSummariseError(null)); + self::assertNull($this->oProcessor->exposeSummariseError('')); + } + + // -------------------------------------------------------------------------- + + /** + * complete() writes a one-block error when the log could not be attached or + * the app's hook objected; there is no phase line to keep. + */ + public function test_a_single_block_error_survives_intact(): void + { + self::assertSame( + 'The error log could not be attached.', + $this->oProcessor->exposeSummariseError('The error log could not be attached.') + ); + } + + // -------------------------------------------------------------------------- + + /** + * validate()'s wording cannot be reused once run() has started: telling an + * admin no accounts were created when some were is worse than saying nothing. + */ + public function test_a_failure_after_accounts_were_created_does_not_claim_none_were(): void + { + $sError = $this->oProcessor->exposeComposeThrowableError( + $this->make([ + 'status' => 'RUNNING', + 'processed_count' => 14, + 'success_count' => 14, + 'error_count' => 0, + ]), + new RuntimeException('Something broke') + ); + + self::assertStringContainsString('14 user accounts had already been created and have been kept', $sError); + self::assertStringNotContainsString('no user accounts were created', $sError); + self::assertStringContainsString('creating user accounts (14 of 1,362 rows processed)', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_a_stalled_job_records_why_it_was_abandoned(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + + $sError = $this->oProcessor->exposeComposeStallError($this->make(), 'no progress was made'); + + self::assertStringContainsString('made no progress and was stopped', $sError); + self::assertStringContainsString('Cause: no progress was made', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_the_summary_is_the_first_line(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + + $sError = $this->oProcessor->exposeComposeValidationError($this->make()); + + self::assertSame( + '2 rows in the CSV could not be validated, so no user accounts were created. Correct them and upload the file again.', + $this->oProcessor->exposeFirstLine($sError) + ); + } + + // -------------------------------------------------------------------------- + + public function test_a_pathological_error_is_bounded(): void + { + $sError = $this->oProcessor->exposeTruncateError(str_repeat('a', Processor::ERROR_MAX_LENGTH * 2)); + + self::assertSame(Processor::ERROR_MAX_LENGTH, mb_strlen(rtrim($sError, '…'))); + self::assertStringEndsWith('…', $sError); + } + + // -------------------------------------------------------------------------- + + /** + * The error quotes cell values from the CSV, so a byte-wise cut would leave + * invalid UTF-8 in the column. + */ + public function test_truncation_does_not_split_a_character(): void + { + $sError = $this->oProcessor->exposeTruncateError(str_repeat('é', Processor::ERROR_MAX_LENGTH * 2)); + + self::assertSame($sError, mb_convert_encoding($sError, 'UTF-8', 'UTF-8')); + self::assertSame(Processor::ERROR_MAX_LENGTH, mb_strlen(rtrim($sError, '…'))); + } + + // -------------------------------------------------------------------------- + + public function test_a_short_error_is_left_alone(): void + { + self::assertSame('Something went wrong.', $this->oProcessor->exposeTruncateError('Something went wrong.')); + } + + // -------------------------------------------------------------------------- + // The technical payload + // -------------------------------------------------------------------------- + + /** + * Mirrors the shape module-queue records for a failed job; see + * Nails\Queue\Service\Manager::buildErrorPayload(). + */ + public function test_the_error_payload_carries_what_is_needed_to_diagnose(): void + { + $aPayload = $this->oProcessor->exposeErrorPayload(new RuntimeException('Something broke', 7)); + + self::assertSame( + ['type', 'message', 'code', 'file', 'line', 'trace', 'occurred_at'], + array_keys($aPayload) + ); + + self::assertSame(RuntimeException::class, $aPayload['type']); + self::assertSame('Something broke', $aPayload['message']); + self::assertSame(7, $aPayload['code']); + self::assertNotEmpty($aPayload['trace']); + } + + // -------------------------------------------------------------------------- + + public function test_a_long_trace_is_truncated(): void + { + $aPayload = $this->oProcessor->exposeErrorPayload( + new RuntimeException(str_repeat('deeply nested ', 500)) + ); + + self::assertLessThanOrEqual( + Processor::ERROR_TRACE_MAX_LENGTH + 1, + mb_strlen($aPayload['trace']) + ); + } + + // -------------------------------------------------------------------------- + // ItemStatus::WARNING + // -------------------------------------------------------------------------- + + /** + * A row whose account was created but whose follow-up failed is neither a + * success nor a failure. 500 silently failed subscriptions must not pass as + * COMPLETE, which is the whole point of the status existing. + */ + public function test_a_job_which_only_warned_finishes_partial(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aWarningSample = $this->importSixWarningSample(); + + $this->oProcessor->exposeComplete($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1361, + 'warning_count' => 1, + 'error_count' => 0, + ])); + + self::assertSame(Status::PARTIAL->value, $this->oModel->lastUpdate()['status']); + } + + // -------------------------------------------------------------------------- + + public function test_a_job_which_neither_errored_nor_warned_finishes_complete(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + + $this->oProcessor->exposeComplete($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1362, + 'warning_count' => 0, + 'error_count' => 0, + ])); + + self::assertSame(Status::COMPLETE->value, $this->oModel->lastUpdate()['status']); + } + + // -------------------------------------------------------------------------- + + /** + * The account exists; only the work which was meant to follow it does not. + * Telling an admin fewer accounts were created than there are is the same + * mistake as telling them none were. + */ + public function test_a_warned_account_counts_among_those_created(): void + { + $sSurvivors = $this->oProcessor->exposeDescribeSurvivors($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 14, + 'success_count' => 12, + 'warning_count' => 2, + 'error_count' => 0, + ])); + + self::assertSame('14 user accounts had already been created and have been kept.', $sSurvivors); + } + + // -------------------------------------------------------------------------- + + public function test_a_single_warned_account_is_worded_in_the_singular(): void + { + $sSurvivors = $this->oProcessor->exposeDescribeSurvivors($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1, + 'success_count' => 0, + 'warning_count' => 1, + 'error_count' => 0, + ])); + + self::assertSame('1 user account had already been created and has been kept.', $sSurvivors); + } + + // -------------------------------------------------------------------------- + + /** + * fail() used to ask for the errors alone, so a job which warned and then + * failed dropped the accounts which need attention from the only record + * which names them. + */ + public function test_a_failed_job_logs_its_warnings_alongside_its_errors(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + $this->oProcessor->aWarningSample = $this->importSixWarningSample(); + + $this->oProcessor->exposeFail( + $this->make(['status' => 'RUNNING', 'success_count' => 4, 'warning_count' => 1]), + 'Something went wrong.' + ); + + self::assertSame( + [[ItemStatus::ERROR, ItemStatus::WARNING]], + $this->oProcessor->aStreamed + ); + } + + // -------------------------------------------------------------------------- + + /** + * A job with nothing to say about any of its rows still must not build a + * log; the warning count is now part of that question. + */ + public function test_a_failure_with_only_warnings_still_builds_a_log(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aWarningSample = $this->importSixWarningSample(); + + $this->oProcessor->exposeFail( + $this->make(['status' => 'RUNNING', 'error_count' => 0, 'warning_count' => 1]), + 'Something went wrong.' + ); + + self::assertSame(20103, $this->oModel->lastUpdate()['log_id']); + } + + // -------------------------------------------------------------------------- + + public function test_the_log_carries_warnings_when_they_are_asked_for(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + $this->oProcessor->aWarningSample = $this->importSixWarningSample(); + + $aItems = $this->oProcessor->exposeStreamItems( + $this->make(), + [ItemStatus::ERROR, ItemStatus::WARNING] + ); + + self::assertSame( + [ItemStatus::ERROR->value, ItemStatus::ERROR->value, ItemStatus::WARNING->value], + array_column($aItems, 'status') + ); + + // ...and only the errors when only they are + self::assertCount( + 2, + $this->oProcessor->exposeStreamItems($this->make(), [ItemStatus::ERROR]) + ); + } + + // -------------------------------------------------------------------------- + // Lifecycle hooks + // -------------------------------------------------------------------------- + + /** + * Wires a hook onto the processor and hands back the service it is on + */ + private function withHook(string $sHook, callable $cHook): ImportServiceStub + { + $oService = new ImportServiceStub(); + $oService->aHooks[$sHook] = $cHook; + $this->oProcessor->oImportService = $oService; + + return $oService; + } + + // -------------------------------------------------------------------------- + + /** + * The one hook which is allowed to refuse the whole job. It fires while + * nothing has been created, so it is deliberately left unwrapped and reaches + * process()'s catch - which means the admin gets the composed reason, with + * the phase and the cause, rather than a bare exception message. + */ + public function test_the_app_may_refuse_the_whole_job_before_anything_is_created(): void + { + $this->oProcessor->sSourcePath = $this->makeValidCsv(); + + $oService = $this->withHook('onImportStart', function (): void { + throw new RuntimeException('the cohort has not been set up yet'); + }); + + $oImport = $this->make([ + 'status' => 'VALIDATING', + 'skip_registered' => true, + 'row_count' => 1, + 'validated_count' => 0, + 'error_count' => 0, + ]); + + // What validate() will read back after recording its progress + $this->oModel->oNext = $this->make([ + 'status' => 'VALIDATING', + 'skip_registered' => true, + 'row_count' => 1, + 'validated_count' => 1, + 'error_count' => 0, + ]); + + $this->oProcessor->process($oImport, 100); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::FAILED->value, $aUpdate['status']); + self::assertStringContainsString('The import stopped unexpectedly', $aUpdate['error']); + self::assertStringContainsString('No user accounts were created.', $aUpdate['error']); + self::assertStringContainsString('the cohort has not been set up yet', $aUpdate['error']); + + // Refused before the job was ever handed to run() + self::assertNotContains( + Status::RUNNING->value, + array_column(array_column($this->oModel->aUpdates, 'data'), 'status') + ); + + self::assertSame(['onImportStart', 'onImportFailed'], $oService->calledHooks()); + } + + // -------------------------------------------------------------------------- + + /** + * By the time complete() runs, every account it was going to create exists, + * so a hook which objects is a note against a finished import - not a + * failed one. + */ + public function test_a_hook_which_objects_to_a_finished_job_cannot_re_brand_it(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + + $this->withHook('onImportComplete', function (): void { + throw new RuntimeException('the cohort roll-up could not be rebuilt'); + }); + + $this->oProcessor->exposeComplete($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1362, + 'warning_count' => 0, + 'error_count' => 0, + ])); + + $aStatuses = array_column(array_column($this->oModel->aUpdates, 'data'), 'status'); + self::assertSame([Status::COMPLETE->value], array_values(array_filter($aStatuses))); + + // ...and the reason is on the job, where the modal and the email read it + $aUpdate = $this->oModel->lastUpdate(); + self::assertSame(['error'], array_keys($aUpdate)); + self::assertStringContainsString('the import itself finished normally', strtolower($aUpdate['error'])); + self::assertStringContainsString('the cohort roll-up could not be rebuilt', $aUpdate['error']); + } + + // -------------------------------------------------------------------------- + + /** + * complete() may already have recorded that the log could not be attached, + * and both reasons matter, so the hook's is appended rather than substituted. + */ + public function test_a_hooks_objection_is_appended_to_a_reason_already_recorded(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'The CDN is unreachable'; + + $this->withHook('onImportComplete', function (): void { + throw new RuntimeException('the cohort roll-up could not be rebuilt'); + }); + + $oImport = $this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1362, + 'warning_count' => 0, + 'error_count' => 0, + ]); + + /** + * The hook is handed the refreshed job, so this is what it - and the + * append - see; without it the recorder would hand back the resource + * this call started with, whose error is still null. + */ + $this->oModel->oNext = $this->make([ + 'status' => 'COMPLETE', + 'processed_count' => 1362, + 'success_count' => 1362, + 'warning_count' => 0, + 'error_count' => 0, + 'error' => 'The error log could not be attached: The CDN is unreachable', + ]); + + $this->oProcessor->exposeComplete($oImport); + + $sError = $this->oModel->lastUpdate()['error']; + + self::assertStringContainsString('The error log could not be attached', $sError); + self::assertStringContainsString('the cohort roll-up could not be rebuilt', $sError); + } + + // -------------------------------------------------------------------------- + + /** + * fail() is reached from process()'s catch-all, so an exception escaping a + * hook here would recurse straight back into it. + */ + public function test_a_hook_which_objects_to_a_failure_does_not_escape(): void + { + $oService = $this->withHook('onImportFailed', function (): void { + throw new RuntimeException('the cohort could not be released'); + }); + + $oImport = $this->oProcessor->exposeFail( + $this->make(['error_count' => 0]), + 'The CSV had no header.' + ); + + self::assertSame(Status::FAILED->value, $this->oModel->lastUpdate()['status']); + self::assertSame(['onImportFailed'], $oService->calledHooks()); + + // Swallowed, but never silently + self::assertNotEmpty(array_filter( + $this->oLogger->aErrors, + fn(string $sLine): bool => str_contains($sLine, 'onImportFailed() failed') + )); + } + + // -------------------------------------------------------------------------- + + /** + * The hook reads the job as it now stands, not as the call which failed it + * found it; an app deciding what to unwind needs the terminal counts. + */ + public function test_the_failure_hook_is_handed_the_job_as_it_now_stands(): void + { + $oService = $this->withHook('onImportFailed', function (): void { + }); + + $this->oModel->oNext = $this->make([ + 'status' => 'FAILED', + 'error_count' => 2, + 'error' => 'The CSV had no header.', + ]); + + $this->oProcessor->exposeFail($this->make(['error_count' => 0]), 'The CSV had no header.'); + + /** @var Resource\User\Import $oGiven */ + $oGiven = $oService->firstCall('onImportFailed')[0]; + + self::assertSame(Status::FAILED, $oGiven->status); + self::assertSame('The CSV had no header.', $oGiven->error); + } + + // -------------------------------------------------------------------------- + // run()'s row hooks + // -------------------------------------------------------------------------- + + /** + * Runs a two row job, with the user model and the hooks stubbed + * + * @return array{0: ImportServiceStub, 1: UserModelStub} + */ + private function runTwoRows(array $aHooks = [], array $aOverrides = []): array + { + $oService = new ImportServiceStub(); + $oUserStub = new UserModelStub(); + + foreach ($aHooks as $sHook => $cHook) { + $oService->aHooks[$sHook] = $cHook; + } + + $this->oProcessor->sSourcePath = $this->makeValidCsv(2); + $this->oProcessor->oImportService = $oService; + $this->oProcessor->oUserModel = $oUserStub; + + $oImport = $this->make(array_merge([ + 'status' => 'RUNNING', + 'skip_registered' => false, + 'row_count' => 2, + 'validated_count' => 2, + 'processed_count' => 0, + 'success_count' => 0, + 'error_count' => 0, + ], $aOverrides)); + + /** + * What run() reads back once it has recorded the chunk; the counts have + * to say the job is finished, or it never reaches complete(). + */ + $this->oModel->oNext = $this->make(array_merge([ + 'status' => 'RUNNING', + 'skip_registered' => false, + 'row_count' => 2, + 'validated_count' => 2, + 'processed_count' => 2, + 'success_count' => 2, + 'error_count' => 0, + ], $aOverrides)); + + $this->oProcessor->exposeRun($oImport, 100); + + return [$oService, $oUserStub]; + } + + // -------------------------------------------------------------------------- + + /** + * Whatever the hook returns is what create() is given; an app which reaches + * for a column the CSV carries but create() would discard has nowhere else + * to put it. + */ + public function test_the_data_hook_decides_what_reaches_create(): void + { + [$oService, $oUserStub] = $this->runTwoRows([ + 'prepareUserData' => function (array $aUserData, $oImport, array $aRow): array { + $aUserData['first_name'] = strtoupper((string) $aRow['username']); + return $aUserData; + }, + ]); + + self::assertSame( + ['USER_1', 'USER_2'], + array_column(array_column($oUserStub->aCreated, 'data'), 'first_name') + ); + + // ...and it saw the row as the CSV had it + self::assertSame('user_1', $oService->firstCall('prepareUserData')[2]['username']); + } + + // -------------------------------------------------------------------------- + + /** + * The additional fields are applied first and the hook runs after them, so + * an app which overrides the hook without calling parent:: still gets what + * it configured - and may overrule it. + */ + public function test_the_data_hook_runs_after_the_additional_fields(): void + { + [, $oUserStub] = $this->runTwoRows( + [ + 'prepareUserData' => function (array $aUserData): array { + $aUserData['salutation'] = strtoupper((string) $aUserData['cohort']); + return $aUserData; + }, + ], + ['additional' => '{"cohort":"autumn"}'] + ); + + self::assertSame('autumn', $oUserStub->aCreated[0]['data']['cohort']); + self::assertSame('AUTUMN', $oUserStub->aCreated[0]['data']['salutation']); + } + + // -------------------------------------------------------------------------- + + /** + * The hook runs before anything has been written, so a row it refuses is a + * plain error - and the job carries on to the next one. + */ + public function test_a_refused_row_errors_and_leaves_no_account(): void + { + [, $oUserStub] = $this->runTwoRows([ + 'prepareUserData' => function (array $aUserData): array { + if ($aUserData['username'] === 'user_1') { + throw new RuntimeException('no cohort could be resolved for this row'); + } + return $aUserData; + }, + ]); + + $aItems = $this->oProcessor->itemsByLine(); + + self::assertSame(ItemStatus::ERROR, $aItems[2]['status']); + self::assertSame('no cohort could be resolved for this row', $aItems[2]['message']); + self::assertNull($aItems[2]['user_id']); + + // No account was attempted for it, and the next row was + self::assertCount(1, $oUserStub->aCreated); + self::assertSame(ItemStatus::SUCCESS, $aItems[3]['status']); + } + + // -------------------------------------------------------------------------- + + /** + * The heart of ItemStatus::WARNING. create() commits before it returns, so + * the account cannot be taken back; the row records that it exists, names + * it, and says what did not happen. + */ + public function test_a_created_account_whose_follow_up_failed_warns_against_its_id(): void + { + [, $oUserStub] = $this->runTwoRows([ + 'afterUserCreate' => function (Resource\User $oUser): void { + if ($oUser->email === 'user1@example.com') { + throw new RuntimeException('the subscription could not be started'); + } + }, + ]); + + $aItems = $this->oProcessor->itemsByLine(); + + self::assertSame(ItemStatus::WARNING, $aItems[2]['status']); + self::assertSame( + 'The account was created, but the subscription could not be started', + $aItems[2]['message'] + ); + + // The user_id is the point: it is the thread back to the account to fix + self::assertSame(100, $aItems[2]['user_id']); + + // The account was still created, and the rest of the job ran + self::assertCount(2, $oUserStub->aCreated); + self::assertSame(ItemStatus::SUCCESS, $aItems[3]['status']); + self::assertSame(101, $aItems[3]['user_id']); + + // ...and it was reported, not swallowed + self::assertNotEmpty(array_filter( + $this->oLogger->aErrors, + fn(string $sLine): bool => str_contains($sLine, 'afterUserCreate() failed') + )); + } + + // -------------------------------------------------------------------------- + + /** + * A warned job has finished; it is only PARTIAL rather than COMPLETE. + */ + public function test_a_job_which_warned_still_reaches_complete(): void + { + $this->runTwoRows( + [ + 'afterUserCreate' => function (): void { + throw new RuntimeException('the subscription could not be started'); + }, + ], + ['success_count' => 1, 'warning_count' => 1] + ); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::PARTIAL->value, $aUpdate['status']); + self::assertNotNull($aUpdate['finished']); + } + + // -------------------------------------------------------------------------- + + /** + * The row is claimed as an error before anything irreversible happens, so a + * create() which refuses a row leaves the reason it gave, not the claim's + * placeholder. + */ + public function test_a_row_create_refuses_keeps_the_reason_create_gave(): void + { + $oService = new ImportServiceStub(); + $oUserStub = new UserModelStub(); + + $oUserStub->sFail = 'false'; + $oUserStub->sCreateError = 'This email is already in use.'; + + $this->oProcessor->sSourcePath = $this->makeValidCsv(1); + $this->oProcessor->oImportService = $oService; + $this->oProcessor->oUserModel = $oUserStub; + + $oImport = $this->make([ + 'status' => 'RUNNING', + 'row_count' => 1, + 'processed_count' => 0, + 'error_count' => 0, + ]); + + $this->oModel->oNext = $oImport; + + $this->oProcessor->exposeRun($oImport, 100); + + $aItems = $this->oProcessor->itemsByLine(); + + self::assertSame(ItemStatus::ERROR, $aItems[2]['status']); + self::assertSame('This email is already in use.', $aItems[2]['message']); + self::assertNull($aItems[2]['user_id']); + + // Nothing was created, so there is nothing for the after-create hook to hear about + self::assertSame(['prepareUserData'], $oService->calledHooks()); + } +} diff --git a/tests/Service/User/Import/ValidatorTest.php b/tests/Service/User/Import/ValidatorTest.php new file mode 100644 index 00000000..46744797 --- /dev/null +++ b/tests/Service/User/Import/ValidatorTest.php @@ -0,0 +1,580 @@ +oValidator = $oValidator; + + $this->mLoginUsing = Config::get('APP_NATIVE_LOGIN_USING'); + } + + // -------------------------------------------------------------------------- + + protected function tearDown(): void + { + foreach ($this->aPaths as $sPath) { + @unlink($sPath); + } + $this->aPaths = []; + + Config::set('APP_NATIVE_LOGIN_USING', $this->mLoginUsing); + } + + // -------------------------------------------------------------------------- + + private function write(string $sContents): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-user-import-test-'); + file_put_contents($sPath, $sContents); + $this->aPaths[] = $sPath; + return $sPath; + } + + // -------------------------------------------------------------------------- + + /** + * Builds a validator over a template an app has amended + * + * @param string[]|null $aKeys + * @param string[]|null $aUniqueKeys + * @param array $aExisting + */ + private function withTemplate( + ?array $aKeys = null, + ?array $aUniqueKeys = null, + array $aExisting = [] + ): array { + $oService = new ImportServiceStub($aKeys, $aUniqueKeys, $aExisting); + $oValidator = new ValidatorWithStub($oService); + + return [$oValidator, $oService]; + } + + // -------------------------------------------------------------------------- + // validateTemplate() + // -------------------------------------------------------------------------- + + public function test_the_stock_template_is_usable(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'BOTH'); + + $this->expectNotToPerformAssertions(); + $this->oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + /** + * The template is the app's to amend, so it can remove the very column an + * account is identified by. Left unguarded that is not noticed until every + * row fails at Model\User::create(). + */ + public function test_a_template_without_the_identity_key_is_refused(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + [$oValidator] = $this->withTemplate( + aKeys: ['username', 'first_name'], + aUniqueKeys: ['username'] + ); + + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('getKeys()'); + + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + public function test_a_template_which_does_not_hold_the_identity_key_unique_is_refused(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + [$oValidator] = $this->withTemplate( + aKeys: ['email', 'first_name'], + aUniqueKeys: [] + ); + + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('getUniqueKeys()'); + + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + /** + * Removing username is legitimate when nobody logs in with it; the guard + * must not forbid customisation, only breakage. + */ + public function test_a_template_may_drop_a_key_the_login_mode_does_not_need(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + [$oValidator] = $this->withTemplate( + aKeys: ['email', 'first_name'], + aUniqueKeys: ['email'] + ); + + $this->expectNotToPerformAssertions(); + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + public function test_both_identity_keys_are_required_of_the_template_when_logging_in_with_either(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'BOTH'); + + [$oValidator] = $this->withTemplate( + aKeys: ['email', 'first_name'], + aUniqueKeys: ['email'] + ); + + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('username'); + + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + // detectRegistered() + // -------------------------------------------------------------------------- + + public function test_registered_values_are_reported_against_their_line(): void + { + [$oValidator] = $this->withTemplate( + aExisting: ['email' => ['a.ahmad6@nhs.net', 'a.botros@nhs.net']] + ); + + self::assertSame( + [ + 2 => ['email: "a.ahmad6@nhs.net" is already registered'], + 3 => ['email: "a.botros@nhs.net" is already registered'], + ], + $oValidator->detectRegistered( + ['email', 'first_name'], + [ + 2 => ['a.ahmad6@nhs.net', 'Ahmad'], + 3 => ['a.botros@nhs.net', 'Botros'], + 4 => ['new@example.com', 'New'], + ] + ) + ); + } + + // -------------------------------------------------------------------------- + + public function test_registration_is_matched_regardless_of_case_or_padding(): void + { + [$oValidator] = $this->withTemplate( + aExisting: ['email' => ['ada@example.com']] + ); + + self::assertArrayHasKey( + 2, + $oValidator->detectRegistered(['email'], [2 => [' ADA@Example.com ']]) + ); + } + + // -------------------------------------------------------------------------- + + public function test_a_blank_value_is_not_looked_up(): void + { + [$oValidator, $oService] = $this->withTemplate( + aExisting: ['email' => ['ada@example.com']] + ); + + self::assertSame([], $oValidator->detectRegistered(['email'], [2 => [''], 3 => [' ']])); + self::assertSame([[]], array_column($oService->aLookups, 'values')); + } + + // -------------------------------------------------------------------------- + + /** + * The whole point of the pass: one lookup per key, not one per row. The + * closures this replaced cost a query - two, on a hit - every line. + */ + public function test_one_lookup_is_made_per_key_however_many_rows(): void + { + [$oValidator, $oService] = $this->withTemplate( + aUniqueKeys: ['email', 'username'], + aExisting: ['email' => ['ada@example.com']] + ); + + $aRows = []; + for ($i = 2; $i <= 500; $i++) { + $aRows[$i] = [sprintf('user%d@example.com', $i), sprintf('user%d', $i)]; + } + + $oValidator->detectRegistered(['email', 'username'], $aRows); + + self::assertSame(['email', 'username'], array_column($oService->aLookups, 'key')); + } + + // -------------------------------------------------------------------------- + + public function test_nothing_is_looked_up_when_no_unique_column_is_present(): void + { + [$oValidator, $oService] = $this->withTemplate(); + + self::assertSame([], $oValidator->detectRegistered(['first_name'], [2 => ['Ada']])); + self::assertSame([], $oService->aLookups); + } + + // -------------------------------------------------------------------------- + + /** + * A value may recur inside a chunk even though the upload's duplicate check + * would reject it across the whole file, and both lines must be reported. + */ + public function test_a_value_repeated_within_the_rows_is_reported_on_every_line(): void + { + [$oValidator] = $this->withTemplate( + aExisting: ['email' => ['ada@example.com']] + ); + + self::assertSame( + [2, 7], + array_keys($oValidator->detectRegistered( + ['email'], + [2 => ['ada@example.com'], 5 => ['new@example.com'], 7 => ['ada@example.com']] + )) + ); + } + + // -------------------------------------------------------------------------- + + /** + * A uniqueness check which quietly answers "none of them" is worse than one + * which fails, so an app adding a unique key must say where it lives. + */ + public function test_a_unique_key_which_cannot_be_looked_up_is_refused(): void + { + $oService = new ImportServiceStub(null, ['staff_number']); + $oService->aUnsupported = ['staff_number']; + $oValidator = new ValidatorWithStub($oService); + + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('staff_number'); + + $oValidator->detectRegistered(['staff_number'], [2 => ['12345']]); + } + + // -------------------------------------------------------------------------- + + public function test_a_valid_header_is_accepted(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + $this->expectNotToPerformAssertions(); + $this->oValidator->validateHeader(['email', 'first_name']); + } + + // -------------------------------------------------------------------------- + + public function test_the_header_need_not_use_every_key(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + $this->expectNotToPerformAssertions(); + $this->oValidator->validateHeader(['email']); + } + + // -------------------------------------------------------------------------- + + /** + * A header may omit almost anything, but not the column an account is + * identified by - Model\User::create() cannot make an account without it, so + * such a file used to fail once per row instead of once. + */ + public function test_a_header_missing_the_identity_column_is_rejected(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('email'); + + $this->oValidator->validateHeader(['first_name', 'last_name']); + } + + // -------------------------------------------------------------------------- + + public function test_the_identity_column_follows_the_login_mode(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'USERNAME'); + + // Username alone is enough... + $this->oValidator->validateHeader(['username']); + + // ...and email alone is not + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('username'); + + $this->oValidator->validateHeader(['email']); + } + + // -------------------------------------------------------------------------- + + /** + * BOTH means both, matching the else branch of Model\User::create(); an + * unset config falls here too. + */ + public function test_both_identity_columns_are_required_when_logging_in_with_either(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'BOTH'); + + $this->oValidator->validateHeader(['email', 'username']); + + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('username'); + + $this->oValidator->validateHeader(['email']); + } + + // -------------------------------------------------------------------------- + + public function test_a_missing_header_is_rejected(): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Missing header row'); + + $this->oValidator->validateHeader([]); + } + + // -------------------------------------------------------------------------- + + public function test_an_unknown_column_is_rejected(): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Header row contains the following invalid values: shoe_size'); + + $this->oValidator->validateHeader(['email', 'shoe_size']); + } + + // -------------------------------------------------------------------------- + + public function test_duplicates_are_detected(): void + { + $sPath = $this->write(implode("\n", [ + 'email,first_name', + 'a@example.com,Ada', + 'b@example.com,Grace', + 'a@example.com,Anita', + '', + ])); + + self::assertSame( + ['Line 4: email: "a@example.com" must only appear once; it is also on line 2'], + $this->oValidator->detectDuplicates($sPath) + ); + } + + // -------------------------------------------------------------------------- + + public function test_duplicate_detection_is_case_insensitive(): void + { + $sPath = $this->write("email\na@example.com\nA@EXAMPLE.COM\n"); + + self::assertCount(1, $this->oValidator->detectDuplicates($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_duplicate_detection_covers_every_unique_key(): void + { + $sPath = $this->write(implode("\n", [ + 'email,username', + 'a@example.com,ada', + 'b@example.com,ada', + 'b@example.com,grace', + '', + ])); + + self::assertSame( + [ + // Reported in line order, not grouped by key + 'Line 3: username: "ada" must only appear once; it is also on line 2', + 'Line 4: email: "b@example.com" must only appear once; it is also on line 3', + ], + $this->oValidator->detectDuplicates($sPath) + ); + } + + // -------------------------------------------------------------------------- + + public function test_blank_values_are_not_duplicates(): void + { + $sPath = $this->write("email,username\na@example.com,\nb@example.com,\n"); + + self::assertSame([], $this->oValidator->detectDuplicates($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_a_file_without_unique_columns_has_no_duplicates(): void + { + $sPath = $this->write("first_name\nAda\nAda\n"); + + self::assertSame([], $this->oValidator->detectDuplicates($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_a_clean_file_has_no_duplicates(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\n"); + + self::assertSame([], $this->oValidator->detectDuplicates($sPath)); + } + // -------------------------------------------------------------------------- + + public function test_form_validation_is_available_without_codeigniter(): void + { + self::assertFalse(function_exists('get_instance')); + self::assertInstanceOf(FormValidation::class, Factory::service('FormValidation')); + } + + // -------------------------------------------------------------------------- + + public function test_clean_rows_have_no_errors(): void + { + $aHeader = ['first_name', 'last_name', 'send_email', 'salutation', 'gender', 'dob']; + $aRows = [ + 2 => ['Ada', 'Lovelace', '1', 'Ms', '', '1815-12-10'], + 3 => ['Grace', 'Hopper', '0', '', '', ''], + ]; + + self::assertSame([], $this->oValidator->validateRows($aHeader, $aRows)); + } + + // -------------------------------------------------------------------------- + + public function test_each_error_class_is_reported_with_its_key_and_message(): void + { + $aHeader = ['send_email', 'salutation', 'last_name', 'dob']; + $aRows = [ + 2 => ['maybe', str_repeat('x', 16), str_repeat('y', 151), '2020-13-45'], + ]; + + self::assertSame( + [ + 2 => [ + 'send_email: This field must be a boolean', + 'salutation: This field is too long, maximum length is 15 characters.', + 'last_name: This field is too long, maximum length is 150 characters.', + 'dob: This field must be a valid date.', + ], + ], + $this->oValidator->validateRows($aHeader, $aRows) + ); + } + + // -------------------------------------------------------------------------- + + public function test_required_fields_fail_when_blank_but_optional_fields_do_not(): void + { + $aHeader = ['send_email', 'dob', 'salutation']; + $aRows = [2 => ['', '', '']]; + + self::assertSame( + [2 => ['send_email: This field is required.']], + $this->oValidator->validateRows($aHeader, $aRows) + ); + } + + // -------------------------------------------------------------------------- + + public function test_only_the_first_failing_rule_per_field_is_reported(): void + { + // send_email is required|is_bool; blank fails required first + $aHeader = ['send_email']; + $aRows = [2 => ['']]; + + self::assertSame( + [2 => ['send_email: This field is required.']], + $this->oValidator->validateRows($aHeader, $aRows) + ); + } + + // -------------------------------------------------------------------------- + + public function test_a_column_count_mismatch_is_reported_alongside_field_errors(): void + { + $aHeader = ['send_email', 'dob']; + $aRows = [2 => ['x']]; + + $aErrors = $this->oValidator->validateRows($aHeader, $aRows); + + self::assertSame(['Row has 1 columns, the header has 2', 'send_email: This field must be a boolean'], $aErrors[2]); + } + + // -------------------------------------------------------------------------- + + public function test_errors_are_keyed_by_line_number(): void + { + $aHeader = ['send_email']; + $aRows = [7 => ['1'], 9 => ['nope'], 12 => ['0']]; + + self::assertSame([9], array_keys($this->oValidator->validateRows($aHeader, $aRows))); + } + + // -------------------------------------------------------------------------- + + public function test_errors_do_not_leak_between_rows(): void + { + $aHeader = ['send_email']; + $aRows = [2 => ['nope'], 3 => ['1']]; + + self::assertSame([2], array_keys($this->oValidator->validateRows($aHeader, $aRows))); + } + + // -------------------------------------------------------------------------- + + public function test_the_gender_rule_uses_the_models_genders(): void + { + $aHeader = ['gender']; + $aRows = [2 => ['not-a-gender']]; + + $aErrors = $this->oValidator->validateRows($aHeader, $aRows); + + self::assertCount(1, $aErrors[2]); + self::assertStringStartsWith('gender: This field must be one of: ', $aErrors[2][0]); + } +} diff --git a/tests/Service/User/ImportTest.php b/tests/Service/User/ImportTest.php new file mode 100644 index 00000000..1b19578a --- /dev/null +++ b/tests/Service/User/ImportTest.php @@ -0,0 +1,193 @@ +oService = $oService; + + $this->mLoginUsing = Config::get('APP_NATIVE_LOGIN_USING'); + } + + // -------------------------------------------------------------------------- + + protected function tearDown(): void + { + Config::set('APP_NATIVE_LOGIN_USING', $this->mLoginUsing); + } + + // -------------------------------------------------------------------------- + + /** + * Counts the closures in a rule set; each one is a potential query per row + * + * @param array $aRules + */ + private function countClosures(array $aRules): int + { + return count(array_filter($aRules, fn($mRule): bool => $mRule instanceof Closure)); + } + + // -------------------------------------------------------------------------- + + /** + * Uniqueness used to be a closure calling getByEmail() for every row - a + * query each, two on a hit. It is now batched; see whichExist(). + */ + public function test_the_email_rules_no_longer_query_per_row(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + $aRules = $this->oService->getValidationRules('email'); + + self::assertSame(0, $this->countClosures($aRules)); + self::assertSame( + [FormValidation::RULE_REQUIRED, FormValidation::RULE_VALID_EMAIL], + array_values($aRules) + ); + } + + // -------------------------------------------------------------------------- + + /** + * The format check remains, and now actually runs: the validator abandons a + * field's remaining rules once one fails, so the uniqueness closure used to + * short-circuit it. + */ + public function test_the_username_rules_keep_only_the_format_check(): void + { + self::assertSame(1, $this->countClosures($this->oService->getValidationRules('username'))); + } + + // -------------------------------------------------------------------------- + + public function test_the_identity_keys_are_unique_keys(): void + { + foreach (['email', 'username'] as $sKey) { + self::assertContains($sKey, $this->oService->getKeys(), $sKey); + self::assertContains($sKey, $this->oService->getUniqueKeys(), $sKey); + } + } + + // -------------------------------------------------------------------------- + + public function test_nothing_is_looked_up_for_an_empty_value_list(): void + { + self::assertSame([], $this->oService->whichExist('email', [])); + self::assertSame([], $this->oService->whichExist('username', [' ', ''])); + } + + // -------------------------------------------------------------------------- + + /** + * An app which adds a unique key must say where the value lives; answering + * "none of them" would let a duplicate through while appearing to check. + * Refused even with nothing to look up, so the misconfiguration cannot hide + * behind an empty file. + */ + public function test_an_unsupported_unique_key_is_refused(): void + { + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('staff_number'); + + $this->oService->whichExist('staff_number', []); + } + + // -------------------------------------------------------------------------- + // Lifecycle hooks + // -------------------------------------------------------------------------- + + /** + * Builds a job to hand the hooks + */ + private function makeImport(): Resource\User\Import + { + return new Resource\User\Import((object) [ + 'id' => 6, + 'object_id' => 2, + 'log_id' => null, + 'additional' => '{"cohort":"2026"}', + 'status' => 'RUNNING', + 'runner' => 'CRON', + 'claim_token' => null, + 'claimed' => null, + 'error' => null, + 'row_count' => 3, + 'validated_count' => 3, + 'processed_count' => 0, + 'success_count' => 0, + 'error_count' => 0, + 'started' => null, + 'finished' => null, + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Every hook ships as a no-op, so an app which overrides one of them changes + * nothing else about how a job runs. + */ + public function test_the_lifecycle_hooks_do_nothing_by_default(): void + { + $oImport = $this->makeImport(); + $oUser = new Resource\User((object) ['id' => 42, 'email' => 'ada@example.com']); + + $aBefore = (array) $oImport; + + // None of them throws, and none of them touches the job it was given + $this->oService->onImportStart($oImport); + $this->oService->afterUserCreate($oUser, ['email' => 'ada@example.com'], $oImport); + $this->oService->onImportComplete($oImport); + $this->oService->onImportFailed($oImport); + + self::assertEquals($aBefore, (array) $oImport); + } + + // -------------------------------------------------------------------------- + + /** + * The guard against a future edit giving this an opinion: whatever the + * processor assembled is what reaches Model\User::create(), untouched. + */ + public function test_the_data_hook_returns_its_input_unchanged(): void + { + $aUserData = [ + 'email' => 'ada@example.com', + 'first_name' => 'Ada', + 'group_id' => 3, + ]; + + self::assertSame( + $aUserData, + $this->oService->prepareUserData( + $aUserData, + $this->makeImport(), + ['email' => 'ada@example.com', 'staff_number' => 'A1234'] + ) + ); + } +} diff --git a/tests/Stub/CdnSpy.php b/tests/Stub/CdnSpy.php new file mode 100644 index 00000000..741d1f32 --- /dev/null +++ b/tests/Stub/CdnSpy.php @@ -0,0 +1,128 @@ + + */ + public array $aDestroyed = []; + + /** + * Object IDs which objectDestroy() should report as failed by returning + * false, mapped onto the error it should then report + * + * @var array + */ + public array $aFailures = []; + + /** + * Object IDs which objectDestroy() should throw for, mapped onto the + * exception message + * + * @var array + */ + public array $aThrows = []; + + // -------------------------------------------------------------------------- + + /** + * Every set of arguments passed to objectCreate(), in order + * + * @var array + */ + public array $aCreated = []; + + /** + * What objectCreate() should hand back + * + * Defaults to a plain stdClass carrying an ID, because that is what the real + * service returns - objectCreate() ends at createObject(), which returns + * Cdn::getObject(), declared `bool|stdClass`. Set to false to model a + * failure, in which case $sObjectCreateError is reported alongside it. + */ + public mixed $mObjectCreateReturn = null; + + /** + * The error objectCreate() should report when it returns false + */ + public ?string $sObjectCreateError = null; + + /** + * The message objectCreate() should throw, if it should throw at all + */ + public ?string $sObjectCreateThrow = null; + + // -------------------------------------------------------------------------- + + public function __construct() + { + // Deliberately does not stand up a driver + } + + // -------------------------------------------------------------------------- + + public function objectCreate($object, $mBucket, $aOptions = [], $bIsStream = false) + { + $this->aCreated[] = [ + 'path' => $object, + 'bucket' => $mBucket, + 'options' => $aOptions, + ]; + + if ($this->sObjectCreateThrow !== null) { + throw new \RuntimeException($this->sObjectCreateThrow); + } + + if ($this->mObjectCreateReturn === false) { + /** + * The real service catches everything, sets an error and returns + * false; it does not throw, which is what makes the reason so easy + * to lose. + */ + if ($this->sObjectCreateError !== null) { + $this->setError($this->sObjectCreateError); + } + + return false; + } + + return $this->mObjectCreateReturn ?? (object) ['id' => 20103]; + } + + // -------------------------------------------------------------------------- + + public function objectDestroy(int|string|Resource\CdnObject|null $object): bool + { + $this->aDestroyed[] = $object; + + if (array_key_exists((int) $object, $this->aThrows)) { + throw new \RuntimeException($this->aThrows[(int) $object]); + } + + if (array_key_exists((int) $object, $this->aFailures)) { + /** + * Mirrors the real service, which reports a missing object, a driver + * failure, and a rolled back transaction by setting an error and + * returning false rather than by throwing. + */ + $this->setError($this->aFailures[(int) $object]); + return false; + } + + return true; + } +} diff --git a/tests/Stub/DatabaseSpy.php b/tests/Stub/DatabaseSpy.php new file mode 100644 index 00000000..7ef28314 --- /dev/null +++ b/tests/Stub/DatabaseSpy.php @@ -0,0 +1,80 @@ + string, 'args' => array] + * + * @var array + */ + public array $aCalls = []; + + /** + * What count_all_results() should report next + */ + public int $iCountAllResults = 0; + + // -------------------------------------------------------------------------- + + public function __construct() + { + // Deliberately does not connect + } + + // -------------------------------------------------------------------------- + + public function __call($sMethod, $aArguments) + { + $this->aCalls[] = [ + 'method' => $sMethod, + 'args' => $aArguments, + ]; + + return match ($sMethod) { + 'count_all_results' => $this->iCountAllResults, + 'update' => true, + default => $this, + }; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the arguments of every call to $sMethod + */ + public function callsTo(string $sMethod): array + { + return array_values( + array_map( + fn(array $aCall) => $aCall['args'], + array_filter( + $this->aCalls, + fn(array $aCall) => $aCall['method'] === $sMethod + ) + ) + ); + } + + // -------------------------------------------------------------------------- + + /** + * The methods which were called, in order + * + * @return string[] + */ + public function methods(): array + { + return array_column($this->aCalls, 'method'); + } +} diff --git a/tests/Stub/ImportModelRecorder.php b/tests/Stub/ImportModelRecorder.php new file mode 100644 index 00000000..165663f6 --- /dev/null +++ b/tests/Stub/ImportModelRecorder.php @@ -0,0 +1,113 @@ + int, 'data' => array] + * + * @var array + */ + public array $aUpdates = []; + + /** + * What update() should report + */ + public bool $bUpdateReturn = true; + + /** + * The error update() should report when it returns false + */ + public ?string $sUpdateError = null; + + /** + * Every job ID passed to deleteItems(), in order + * + * @var int[] + */ + public array $aItemsDeleted = []; + + /** + * What getById() should hand back; null makes the processor keep the + * resource it already had + */ + public ?Resource\User\Import $oNext = null; + + // -------------------------------------------------------------------------- + + /** + * The counts the model derives are read through here, so a recorder can + * answer them without a server; see Model\User\Import::countItems() + */ + public DatabaseSpy $oDbSpy; + + // -------------------------------------------------------------------------- + + public function __construct() + { + $this->oDbSpy = new DatabaseSpy(); + parent::__construct(); + } + + // -------------------------------------------------------------------------- + + protected function getDb(): Database + { + return $this->oDbSpy; + } + + // -------------------------------------------------------------------------- + + public function update($iId, array $aData = []): bool + { + $this->aUpdates[] = [ + 'id' => (int) $iId, + 'data' => $aData, + ]; + + if (!$this->bUpdateReturn && $this->sUpdateError !== null) { + $this->setError($this->sUpdateError); + } + + return $this->bUpdateReturn; + } + + // -------------------------------------------------------------------------- + + public function getById(?int $iId, array $aData = []) + { + return $this->oNext; + } + + // -------------------------------------------------------------------------- + + public function deleteItems(int $iId): bool + { + $this->aItemsDeleted[] = $iId; + return true; + } + + // -------------------------------------------------------------------------- + + /** + * The data of the last update, or null if there has not been one + */ + public function lastUpdate(): ?array + { + $aUpdate = end($this->aUpdates); + return $aUpdate === false ? null : $aUpdate['data']; + } +} diff --git a/tests/Stub/ImportModelWithSpy.php b/tests/Stub/ImportModelWithSpy.php new file mode 100644 index 00000000..c08c218a --- /dev/null +++ b/tests/Stub/ImportModelWithSpy.php @@ -0,0 +1,34 @@ +oSpy; + } + + // -------------------------------------------------------------------------- + + protected function getCdn(): Cdn + { + return $this->oCdnSpy ?? new CdnSpy(); + } +} diff --git a/tests/Stub/ImportServiceStub.php b/tests/Stub/ImportServiceStub.php new file mode 100644 index 00000000..a81c007f --- /dev/null +++ b/tests/Stub/ImportServiceStub.php @@ -0,0 +1,193 @@ + $aExisting Values already taken, keyed by unique key + */ + public function __construct( + private ?array $aKeys = null, + private ?array $aUniqueKeys = null, + private array $aExisting = [] + ) { + } + + // -------------------------------------------------------------------------- + + /** + * Every call to whichExist(), in order: ['key' => string, 'values' => string[]] + * + * @var array + */ + public array $aLookups = []; + + /** + * Unique keys which whichExist() should refuse, modelling an app which has + * added a key without saying where it lives + * + * @var string[] + */ + public array $aUnsupported = []; + + /** + * What each lifecycle hook should do, keyed by hook name; null leaves the + * inherited no-op in place. This is the shape an app subclass takes - one + * override per hook - with the body made settable per test. + * + * @var array + */ + public array $aHooks = []; + + /** + * Every hook call, in order: ['hook' => string, 'args' => array] + * + * @var array + */ + public array $aHookCalls = []; + + // -------------------------------------------------------------------------- + + public function getKeys(): array + { + return $this->aKeys ?? parent::getKeys(); + } + + // -------------------------------------------------------------------------- + + public function getUniqueKeys(): array + { + return $this->aUniqueKeys ?? parent::getUniqueKeys(); + } + + // -------------------------------------------------------------------------- + + public function whichExist(string $sKey, array $aValues): array + { + $this->aLookups[] = [ + 'key' => $sKey, + 'values' => $aValues, + ]; + + if (in_array($sKey, $this->aUnsupported, true)) { + // Mirrors the real service's refusal to answer for a key it cannot look up + return parent::whichExist($sKey, $aValues); + } + + return array_values( + array_intersect( + array_map('strtolower', $this->aExisting[$sKey] ?? []), + array_map('strtolower', $aValues) + ) + ); + } + + // -------------------------------------------------------------------------- + // Lifecycle hooks + // -------------------------------------------------------------------------- + + public function onImportStart(Resource\User\Import $oImport): void + { + $this->fire('onImportStart', [$oImport]); + } + + // -------------------------------------------------------------------------- + + public function prepareUserData(array $aUserData, Resource\User\Import $oImport, array $aRow): array + { + $mReturn = $this->fire('prepareUserData', [$aUserData, $oImport, $aRow]); + + /** + * Falls back to the parent rather than to $aUserData: an unset hook has + * to exercise the real no-op, which is what guards against a future edit + * giving it an opinion. + */ + return $mReturn ?? parent::prepareUserData($aUserData, $oImport, $aRow); + } + + // -------------------------------------------------------------------------- + + public function afterUserCreate( + Resource\User $oUser, + array $aUserData, + Resource\User\Import $oImport + ): void { + $this->fire('afterUserCreate', [$oUser, $aUserData, $oImport]); + } + + // -------------------------------------------------------------------------- + + public function onImportComplete(Resource\User\Import $oImport): void + { + $this->fire('onImportComplete', [$oImport]); + } + + // -------------------------------------------------------------------------- + + public function onImportFailed(Resource\User\Import $oImport): void + { + $this->fire('onImportFailed', [$oImport]); + } + + // -------------------------------------------------------------------------- + + /** + * Records the call and runs whatever the test set for the hook + * + * @param array $aArgs + * + * @return mixed Whatever the hook returned, or null if none was set + */ + private function fire(string $sHook, array $aArgs): mixed + { + $this->aHookCalls[] = [ + 'hook' => $sHook, + 'args' => $aArgs, + ]; + + $cHook = $this->aHooks[$sHook] ?? null; + + return $cHook === null ? null : $cHook(...$aArgs); + } + + // -------------------------------------------------------------------------- + + /** + * The hooks which have been called, in order + * + * @return string[] + */ + public function calledHooks(): array + { + return array_column($this->aHookCalls, 'hook'); + } + + // -------------------------------------------------------------------------- + + /** + * The arguments of the first call to a hook, or null if it was not called + * + * @return array|null + */ + public function firstCall(string $sHook): ?array + { + foreach ($this->aHookCalls as $aCall) { + if ($aCall['hook'] === $sHook) { + return $aCall['args']; + } + } + + return null; + } +} diff --git a/tests/Stub/LoggerSpy.php b/tests/Stub/LoggerSpy.php new file mode 100644 index 00000000..eaa6b282 --- /dev/null +++ b/tests/Stub/LoggerSpy.php @@ -0,0 +1,68 @@ +aInfo[] = (string) $sLine; + return $this; + } + + // -------------------------------------------------------------------------- + + public function warning($sLine = ''): self + { + $this->aWarnings[] = (string) $sLine; + return $this; + } + + // -------------------------------------------------------------------------- + + public function error($sLine = ''): self + { + $this->aErrors[] = (string) $sLine; + return $this; + } + + // -------------------------------------------------------------------------- + + /** + * Every line recorded, whatever the level + * + * @return string[] + */ + public function all(): array + { + return array_merge($this->aInfo, $this->aWarnings, $this->aErrors); + } +} diff --git a/tests/Stub/ProcessorWithSpies.php b/tests/Stub/ProcessorWithSpies.php new file mode 100644 index 00000000..ffa7659e --- /dev/null +++ b/tests/Stub/ProcessorWithSpies.php @@ -0,0 +1,330 @@ + int, 'message' => string|null], ...] + * + * @var array + */ + public array $aSample = []; + + /** + * The job's warned rows, in the same shape as $aSample + * + * @var array + */ + public array $aWarningSample = []; + + /** + * The import service the hooks are fired against; the real one unless a + * test supplies its own + */ + public ?ImportService $oImportService = null; + + /** + * The user model accounts are created through; the real one unless a test + * supplies its own + */ + public ?UserModel $oUserModel = null; + + /** + * The statuses each call to streamItems() asked for, in order + * + * @var array + */ + public array $aStreamed = []; + + /** + * Every item claimed by run(), keyed by the ID it was given: + * ['line' => int, 'status' => ItemStatus, 'message' => ?string, 'user_id' => ?int] + * + * @var array + */ + public array $aItems = []; + + /** + * Lines getHandledLines() should report as already dealt with + * + * @var int[] + */ + public array $aHandled = []; + + // -------------------------------------------------------------------------- + + public function __construct( + public ImportModelRecorder $oModelStub, + public CdnSpy $oCdnSpy, + public LoggerSpy $oLoggerSpy + ) { + } + + // -------------------------------------------------------------------------- + // Seams + // -------------------------------------------------------------------------- + + protected function getModel(): ImportModel + { + return $this->oModelStub; + } + + protected function getCdn(): Cdn + { + return $this->oCdnSpy; + } + + protected function getLogger(): Logger + { + return $this->oLoggerSpy; + } + + protected function getSourcePath(Resource\User\Import $oImport): string + { + return $this->sSourcePath ?? parent::getSourcePath($oImport); + } + + protected function refresh(Resource\User\Import $oImport): Resource\User\Import + { + return $this->oModelStub->oNext ?? $oImport; + } + + /** + * The real implementation needs module-cdn's MetaData interfaces, which the + * copy this module vendors for its own tests does not carry - the same + * staleness which makes phpstan report src/Cdn/MetaData/SystemKey/*. The + * keys are not what these tests are about, so they are left out. + */ + protected function getLogMetaData(Resource\User\Import $oImport): array + { + return []; + } + + protected function getImportService(): ImportService + { + return $this->oImportService ?? parent::getImportService(); + } + + protected function getUserModel(): UserModel + { + return $this->oUserModel ?? parent::getUserModel(); + } + + /** + * The item table stands in for itself: the claim-then-resolve contract is + * what run() is built on, and asserting the outcome of a row means being + * able to read it back without a database. + */ + protected function recordItem( + Resource\User\Import $oImport, + int $iLine, + ItemStatus $oStatus, + ?string $sMessage = null, + ?int $iUserId = null + ): ?int { + + $iItemId = count($this->aItems) + 1; + + $this->aItems[$iItemId] = [ + 'line' => $iLine, + 'status' => $oStatus, + 'message' => $sMessage, + 'user_id' => $iUserId, + ]; + + return $iItemId; + } + + protected function resolveItem( + int $iItemId, + ItemStatus $oStatus, + ?string $sMessage = null, + ?int $iUserId = null + ): void { + + $this->aItems[$iItemId] = array_merge( + $this->aItems[$iItemId] ?? ['line' => 0], + [ + 'status' => $oStatus, + 'message' => $sMessage, + 'user_id' => $iUserId, + ] + ); + } + + protected function getHandledLines(Resource\User\Import $oImport, array $aLines): array + { + return array_values(array_intersect($this->aHandled, $aLines)); + } + + protected function sampleErrors(Resource\User\Import $oImport, int $iLimit): array + { + return array_slice($this->aSample, 0, $iLimit); + } + + /** + * @param ItemStatus[]|null $aStatuses + */ + protected function streamItems(Resource\User\Import $oImport, ?array $aStatuses = null): Generator + { + $this->aStreamed[] = $aStatuses; + + $aItems = array_merge( + array_map( + fn(array $aItem): array => $aItem + ['status' => ItemStatus::ERROR], + $this->aSample + ), + array_map( + fn(array $aItem): array => $aItem + ['status' => ItemStatus::WARNING], + $this->aWarningSample + ) + ); + + foreach ($aItems as $aItem) { + + if (!empty($aStatuses) && !in_array($aItem['status'], $aStatuses, true)) { + continue; + } + + yield $aItem['line'] => [ + 'id' => null, + 'status' => $aItem['status']->value, + 'message' => $aItem['message'], + ]; + } + } + + // -------------------------------------------------------------------------- + // Exposed for assertion + // -------------------------------------------------------------------------- + + public function exposeUploadLog(Resource\User\Import $oImport, iterable $aItems): int + { + return $this->uploadLog($oImport, $aItems); + } + + /** + * @param ItemStatus[]|null $aStatuses + */ + public function exposeAttachLog(Resource\User\Import $oImport, ?array $aStatuses = null): array + { + return $this->attachLog($oImport, $aStatuses); + } + + /** + * @param ItemStatus[]|null $aStatuses + * + * @return array + */ + public function exposeStreamItems(Resource\User\Import $oImport, ?array $aStatuses = null): array + { + return iterator_to_array($this->streamItems($oImport, $aStatuses), true); + } + + public function exposeDescribeSurvivors(Resource\User\Import $oImport): ?string + { + return $this->describeSurvivors($oImport); + } + + public function exposeBegin(Resource\User\Import $oImport): Resource\User\Import + { + return $this->begin($oImport); + } + + public function exposeRun(Resource\User\Import $oImport, int $iLimit): Resource\User\Import + { + return $this->run($oImport, $iLimit); + } + + /** + * The items run() recorded, in line order + * + * @return array + */ + public function itemsByLine(): array + { + $aItems = []; + + foreach ($this->aItems as $aItem) { + $aItems[$aItem['line']] = $aItem; + } + + ksort($aItems); + + return $aItems; + } + + public function exposeComplete(Resource\User\Import $oImport): Resource\User\Import + { + return $this->complete($oImport); + } + + public function exposeFail( + Resource\User\Import $oImport, + string $sError, + ?Throwable $e = null + ): Resource\User\Import { + return $this->fail($oImport, $sError, $e); + } + + public function exposeComposeValidationError(Resource\User\Import $oImport): string + { + return $this->composeValidationError($oImport); + } + + public function exposeComposeThrowableError(Resource\User\Import $oImport, Throwable $e): string + { + return $this->composeThrowableError($oImport, $e); + } + + public function exposeComposeStallError(Resource\User\Import $oImport, string $sReason): string + { + return $this->composeStallError($oImport, $sReason); + } + + public function exposeErrorPayload(Throwable $e): array + { + return $this->errorPayload($e); + } + + public function exposeTruncateError(string $sError): string + { + return $this->truncateError($sError); + } + + public function exposeFirstLine(string $sError): string + { + return $this->firstLine($sError); + } + + public function exposeSummariseError(?string $sError): ?string + { + return $this->summariseError($sError); + } +} diff --git a/tests/Stub/UserModelStub.php b/tests/Stub/UserModelStub.php new file mode 100644 index 00000000..577aede0 --- /dev/null +++ b/tests/Stub/UserModelStub.php @@ -0,0 +1,67 @@ + array, 'send_email' => bool] + * + * @var array + */ + public array $aCreated = []; + + /** + * The ID to give the next account; incremented per call + */ + public int $iNextId = 100; + + /** + * What create() should do instead of succeeding: 'false' to report a + * failure the way the real model does, 'throw' to fall over + */ + public ?string $sFail = null; + + /** + * The error create() should leave behind when it returns false + */ + public ?string $sCreateError = 'The account could not be created'; + + // -------------------------------------------------------------------------- + + public function create(array $data = [], $bSendWelcome = true) + { + $this->aCreated[] = [ + 'data' => $data, + 'send_email' => (bool) $bSendWelcome, + ]; + + if ($this->sFail === 'throw') { + throw new RuntimeException('the account could not be written'); + + } elseif ($this->sFail === 'false') { + + if ($this->sCreateError !== null) { + $this->setError($this->sCreateError); + } + + return false; + } + + return new Resource\User((object) [ + 'id' => $this->iNextId++, + 'email' => $data['email'] ?? null, + ]); + } +} diff --git a/tests/Stub/ValidatorWithStub.php b/tests/Stub/ValidatorWithStub.php new file mode 100644 index 00000000..cc962864 --- /dev/null +++ b/tests/Stub/ValidatorWithStub.php @@ -0,0 +1,24 @@ +oImportService; + } +}