diff --git a/.gitignore b/.gitignore index 021a42aa..be155a64 100644 --- a/.gitignore +++ b/.gitignore @@ -248,3 +248,6 @@ $RECYCLE.BIN/ *.lnk # End of https://www.gitignore.io/api/linux,macos,windows,phpstorm,visualstudiocode,composer,node + +# PHPUnit +.phpunit.result.cache diff --git a/admin/views/Import/index.php b/admin/views/Import/index.php index 649c3beb..28184005 100644 --- a/admin/views/Import/index.php +++ b/admin/views/Import/index.php @@ -6,8 +6,24 @@ /** * @var array $additionalFields + * @var bool $bTemplateUnusable */ +/** + * The controller has already reported why; offering a form which cannot work + * would only invite an upload that is certain to be rejected. + */ +if (!empty($bTemplateUnusable)) { + ?> +
+

+ User import is unavailable until the template is corrected. +

+
+
@@ -29,6 +45,8 @@

Please note: The CSV you supply should be in the correct format, as per the template which you can download above. Remember to include the header rows describing each column. +
Every row is validated when you upload; if any of them cannot be imported the file is rejected + and nothing is created, so you can correct it and try again.

[ - 'text' => 'Preview', - 'name' => 'action', - 'value' => 'preview', + 'text' => 'Upload & Preview', ], ]); echo form_close(); ?> +
diff --git a/admin/views/Import/preview.php b/admin/views/Import/preview.php index 9e17a188..0a15304a 100644 --- a/admin/views/Import/preview.php +++ b/admin/views/Import/preview.php @@ -1,65 +1,336 @@ $aRegistered */ +$bIsDraft = $oImport->status === Status::DRAFT; +$iColumns = count($aKeys) + 1; +$iRowCount = (int) ($oImport->row_count ?? 0); + +/** + * Rows whose email or username already belongs to an account. The CSV is not + * wrong - these rows are simply redundant - so rather than rejecting the file + * the admin is offered the chance to import the rest without them. + */ +$aRegistered = $aRegistered ?? []; +$iRegistered = count($aRegistered); +$iImportable = max(0, $iRowCount - $iRegistered); + +/** + * The confirmation can be exact rather than hedging with "up to": both routes + * to it - ticking the skip checkbox below, or Continue on the warning an + * unticked Import opens - agree to skip exactly these rows. + */ +$sConfirmBody = sprintf( + '%s %s will be created in the background%s. This cannot be undone.', + number_format($iImportable), + $iImportable === 1 ? 'user account' : 'user accounts', + $iRegistered + ? sprintf( + ', and %s %s will be skipped', + number_format($iRegistered), + $iRegistered === 1 ? 'row' : 'rows' + ) + : '' +); + +/** + * The warning an unticked Import opens, reusing the alert's own phrasing so the + * modal does not introduce a second account of the same problem. Continue is an + * implied tick, so the copy names both halves of what it agrees to. + */ +$sWarnTitle = sprintf( + '%s %s already registered', + number_format($iRegistered), + $iRegistered === 1 ? 'row is' : 'rows are' +); + +$sWarnBody = sprintf( + 'An account already exists for %s of the rows in this CSV, so %s cannot be imported. ' + . 'Continue to import the remaining %s and skip %s, or cancel and correct your CSV.', + number_format($iRegistered), + $iRegistered === 1 ? 'it' : 'they', + number_format($iImportable), + $iRegistered === 1 ? 'it' : 'them' +); + +/** + * Delete is a plain button which UserImport turns into a + * DELETE /api/auth/import/{id}; see \Nails\Auth\Api\Controller\Import. Unlike + * the approve form it does not degrade - with the bundle broken it does nothing + * - which is the same bargain the preview table below already makes. + * + * `type="button"` is load bearing: on a draft this sits inside the approve + * form, and the default `submit` would start the import. + * + * The status and the created count are handed over so that the confirmation + * copy can be composed in one place, in JS, rather than written out again here. + */ +/** + * Details is a plain button which UserImport turns into a modal; see + * assets/js/components/UserImport.js. Offered only when there is something to + * show - a stored reason, or rows which failed or warned - so a clean import + * does not get a button which opens an empty box. + */ +$bHasDetails = $oImport->status->isTerminal() + && ($oImport->error || $oImport->error_count || $oImport->warning_count); + +$sDetailsButton = $bHasDetails + ? sprintf( + '', + $oImport->id + ) + : null; + +$sDeleteButton = $oImport->status->isDeletable() + ? sprintf( + '', + implode(' ', [ + 'data-import-id="' . $oImport->id . '"', + 'data-delete-url="' . siteUrl('api/auth/import/' . $oImport->id) . '"', + 'data-redirect="' . siteUrl($sListUrl) . '"', + 'data-status="' . $oImport->status->value . '"', + // Warned rows have accounts too; see Enum\User\Import\ItemStatus + 'data-created-count="' . ($oImport->success_count + $oImport->warning_count) . '"', + ]) + ) + : null; + +/** + * A draft can be imported or deleted; anything else which is deletable gets + * Delete alone, with nothing to save. A job a runner holds gets no controls at + * all, so the bar is absent entirely for those two statuses. + */ +if ($bIsDraft) { + $aControls = [ + 'save' => ['text' => 'Import'], + 'html' => ['right' => $sDeleteButton], + ]; +} elseif ($sDeleteButton || $sDetailsButton) { + $aControls = [ + 'save' => ['enabled' => false], + 'html' => ['right' => trim($sDetailsButton . ' ' . $sDeleteButton)], + ]; +} else { + $aControls = null; +} + +/** + * Mirrors how module-admin normalises a column label into a cell class; see + * module-admin/admin/views/DefaultController/index.php + */ +$fFieldClass = function (string $sLabel): string { + $sLabel = strtolower($sLabel); + $sLabel = preg_replace('/[^a-z0-9 \-_]/', '', $sLabel); + return 'field field--' . str_replace([' ', '_'], '-', $sLabel); +}; + ?>
-
- Please review the following data -
Your CSV has been processed, and the following values have been ascertained. Please verify them, and when - happy to continue, click "Import" below. -
- - - - - - - - + +
+ Please review the following data +
Your CSV has been uploaded and every row has been validated. Please verify the values below, and + when happy to continue, click "Import" below. The import runs in the background; you can leave this page + once it has started. +
+ +
+ + + already registered + +
An account already exists for the following, so + cannot be imported. Tick the box below to import the + remaining , or correct your CSV and upload it again. +
$aLineErrors) { + foreach ($aLineErrors as $sLineError) { + echo htmlspecialchars( + sprintf('Line %d: %s', $iLine, $sLineError), + ENT_QUOTES + ) . '
'; + } + } + + ?> +
+ -
- - - +

+ +

+ '; - foreach ($aKeys as $sKey) { + ?> + +
+ This import is status->value?> and can no longer be changed. + error) { ?> -
+
error, 2)[0], ENT_QUOTES)?> '; + ?> +

+ + + +
+ > +
+ + + -
-
—'?>
+
+
+ + + + + + + + + + + + + + +
Line
+ + + Loading + +
+
+
+ [ - 'text' => 'Import', - 'name' => 'action', - 'value' => 'import', - ], - ]); + if ($aControls) { + echo Helper::floatingControls($aControls); + } + + if ($bIsDraft) { + echo form_close(); + } ?> - diff --git a/assets/css/admin.min.css b/assets/css/admin.min.css index 961d2d50..d87c9558 100644 --- a/assets/css/admin.min.css +++ b/assets/css/admin.min.css @@ -1 +1 @@ -.group-accounts.all table{margin-bottom:1em}.group-accounts.all table th.id,.group-accounts.all table th.group,.group-accounts.all table td.id,.group-accounts.all table td.group{text-align:center}.group-accounts.all table th.id,.group-accounts.all table td.id{width:50px;min-width:50px}.group-accounts.all table th.group,.group-accounts.all table td.group{width:150px;min-width:150px}.group-accounts.all table th.actions,.group-accounts.all table td.actions{width:225px;min-width:225px;text-align:center}.group-accounts.all table td.details img.profile-img{float:left;margin-right:10px;width:65px;height:65px;background:#ccc;border:1px solid #ccc;border-radius:2px}.group-accounts.all table td.details img.verified{margin-left:5px;width:10px;height:10px;position:relative;top:2px}.group-accounts.all table td.id,.group-accounts.all table td.group{vertical-align:middle;color:#666}.group-accounts.all table td.profile.zero{color:#ccc}.group-accounts.all table td.actions .not-editable{color:#aaa}.group-accounts.all table td .awesome{margin-right:5px;margin-bottom:5px;padding:8px 5px 6px 5px}.group-accounts.all table td div{float:left}.group-accounts.all table td div small{display:block}.group-accounts.create #user-group-descriptions li,.group-accounts.create #user-group-pwrules li{margin:0}.group-accounts.edit .social .icon{float:left;border:1px solid #999;border-radius:3px;background:#f3f3f3;padding:5px;margin-right:15px;padding-left:42px;line-height:16px;background-position:5px center;background-repeat:no-repeat;word-wrap:break-word}.group-accounts.edit .social p{margin-top:5px}.group-accounts.edit .uploads ul{max-height:500px;overflow:auto}.group-accounts.edit .uploads ul li{vertical-align:middle;display:block;margin-bottom:0}.group-accounts.edit .uploads ul li a{padding:1em;display:block;position:relative;min-height:51px;background:#efefef;background-clip:padding-box;text-decoration:none}.group-accounts.edit .uploads ul li a small{display:block}.group-accounts.edit .uploads ul li a.image{padding-left:55px}.group-accounts.edit .uploads ul li a.image img{position:absolute;left:5px;top:5px;width:35px;height:35px;border:1px solid #ccc;margin:0;margin-right:10px;vertical-align:middle;padding:2px;background:#fff;border-radius:2px}.group-accounts.edit .uploads ul li.no-data{color:#777}.group-accounts.edit table.emails th.email input,.group-accounts.edit table.emails td.email input{width:100%;box-sizing:border-box}.group-accounts.edit table.emails th.actions,.group-accounts.edit table.emails td.actions{width:220px}.group-accounts.groups table th,.group-accounts.groups table td{vertical-align:middle}.group-accounts.groups table th.actions,.group-accounts.groups table th.actions,.group-accounts.groups table td.actions,.group-accounts.groups table td.actions{min-width:220px;text-align:center}.group-accounts.groups table th.default,.group-accounts.groups table th.default,.group-accounts.groups table td.default,.group-accounts.groups table td.default{width:50px;text-align:center}.group-accounts.groups table th.default span,.group-accounts.groups table th.default span,.group-accounts.groups table td.default span,.group-accounts.groups table td.default span{font-size:1.5em}.group-accounts.groups.edit .permission-groups .permission-group th.enabled{width:50px}.group-accounts.groups.edit .permission-groups .permission-group td{cursor:pointer}.group-accounts.groups.edit .permission-groups .permission-group td.enabled{padding:0}.group-accounts.groups.edit .permission-groups .permission-group td.enabled label{width:100%;height:100%;cursor:pointer;line-height:40px}.group-accounts.groups.edit .search{border:none;border-bottom:1px solid #efefef}.group-accounts.groups.edit .search .search-text{margin:0}.group-accounts.merge.merge-preview table{margin-bottom:1.5em}.group-accounts.merge.merge-preview table th.userId,.group-accounts.merge.merge-preview table td.userId{text-align:center;width:50px}.group-accounts.merge.merge-preview table th.user-cell,.group-accounts.merge.merge-preview table td.user-cell{width:auto}.group-accounts.merge.merge-preview table th.tableRows,.group-accounts.merge.merge-preview table td.tableRows{text-align:center;width:100px}.group-accounts.change-group table th.userId,.group-accounts.change-group table td.userId{text-align:center;width:50px} +.group-accounts.all table{margin-bottom:1em}.group-accounts.all table th.id,.group-accounts.all table th.group,.group-accounts.all table td.id,.group-accounts.all table td.group{text-align:center}.group-accounts.all table th.id,.group-accounts.all table td.id{width:50px;min-width:50px}.group-accounts.all table th.group,.group-accounts.all table td.group{width:150px;min-width:150px}.group-accounts.all table th.actions,.group-accounts.all table td.actions{width:225px;min-width:225px;text-align:center}.group-accounts.all table td.details img.profile-img{float:left;margin-right:10px;width:65px;height:65px;background:#ccc;border:1px solid #ccc;border-radius:2px}.group-accounts.all table td.details img.verified{margin-left:5px;width:10px;height:10px;position:relative;top:2px}.group-accounts.all table td.id,.group-accounts.all table td.group{vertical-align:middle;color:#666}.group-accounts.all table td.profile.zero{color:#ccc}.group-accounts.all table td.actions .not-editable{color:#aaa}.group-accounts.all table td .awesome{margin-right:5px;margin-bottom:5px;padding:8px 5px 6px 5px}.group-accounts.all table td div{float:left}.group-accounts.all table td div small{display:block}.group-accounts.create #user-group-descriptions li,.group-accounts.create #user-group-pwrules li{margin:0}.group-accounts.edit .social .icon{float:left;border:1px solid #999;border-radius:3px;background:#f3f3f3;padding:5px;margin-right:15px;padding-left:42px;line-height:16px;background-position:5px center;background-repeat:no-repeat;word-wrap:break-word}.group-accounts.edit .social p{margin-top:5px}.group-accounts.edit .uploads ul{max-height:500px;overflow:auto}.group-accounts.edit .uploads ul li{vertical-align:middle;display:block;margin-bottom:0}.group-accounts.edit .uploads ul li a{padding:1em;display:block;position:relative;min-height:51px;background:#efefef;background-clip:padding-box;text-decoration:none}.group-accounts.edit .uploads ul li a small{display:block}.group-accounts.edit .uploads ul li a.image{padding-left:55px}.group-accounts.edit .uploads ul li a.image img{position:absolute;left:5px;top:5px;width:35px;height:35px;border:1px solid #ccc;margin:0;margin-right:10px;vertical-align:middle;padding:2px;background:#fff;border-radius:2px}.group-accounts.edit .uploads ul li.no-data{color:#777}.group-accounts.edit table.emails th.email input,.group-accounts.edit table.emails td.email input{width:100%;box-sizing:border-box}.group-accounts.edit table.emails th.actions,.group-accounts.edit table.emails td.actions{width:220px}.group-accounts.groups table th,.group-accounts.groups table td{vertical-align:middle}.group-accounts.groups table th.actions,.group-accounts.groups table th.actions,.group-accounts.groups table td.actions,.group-accounts.groups table td.actions{min-width:220px;text-align:center}.group-accounts.groups table th.default,.group-accounts.groups table th.default,.group-accounts.groups table td.default,.group-accounts.groups table td.default{width:50px;text-align:center}.group-accounts.groups table th.default span,.group-accounts.groups table th.default span,.group-accounts.groups table td.default span,.group-accounts.groups table td.default span{font-size:1.5em}.group-accounts.groups.edit .permission-groups .permission-group th.enabled{width:50px}.group-accounts.groups.edit .permission-groups .permission-group td{cursor:pointer}.group-accounts.groups.edit .permission-groups .permission-group td.enabled{padding:0}.group-accounts.groups.edit .permission-groups .permission-group td.enabled label{width:100%;height:100%;cursor:pointer;line-height:40px}.group-accounts.groups.edit .search{border:none;border-bottom:1px solid #efefef}.group-accounts.groups.edit .search .search-text{margin:0}.group-accounts.merge.merge-preview table{margin-bottom:1.5em}.group-accounts.merge.merge-preview table th.userId,.group-accounts.merge.merge-preview table td.userId{text-align:center;width:50px}.group-accounts.merge.merge-preview table th.user-cell,.group-accounts.merge.merge-preview table td.user-cell{width:auto}.group-accounts.merge.merge-preview table th.tableRows,.group-accounts.merge.merge-preview table td.tableRows{text-align:center;width:100px}.group-accounts.change-group table th.userId,.group-accounts.change-group table td.userId{text-align:center;width:50px}.badge.badge--default{background-color:#555}.badge.badge--info{background-color:#31708f}.badge.badge--success{background-color:#3c763d}.badge.badge--warning{background-color:#8a6d3b}.badge.badge--danger{background-color:#a94442}@-webkit-keyframes user-import-spin{to{transform:rotate(360deg)}}@keyframes user-import-spin{to{transform:rotate(360deg)}}.user-import-spinner{display:inline-block;width:.85em;height:.85em;vertical-align:-0.1em;border:2px solid currentColor;border-right-color:rgba(0,0,0,0);border-radius:50%;-webkit-animation:user-import-spin .75s linear infinite;animation:user-import-spin .75s linear infinite}.user-import-details p{margin:0 0 .5rem}.user-import-details__log,.user-import-details__rows{max-height:40vh;overflow:auto;margin:0 0 .75rem;padding:.75rem;font-size:12px;line-height:1.5;white-space:pre-wrap;word-break:break-word;background-color:#f5f5f5;border:1px solid #d9d9d9;border-radius:4px}.module-auth.import th.actions,.module-auth.import td.actions{width:275px;min-width:275px}.module-auth.import--preview #user-import-preview tbody{transition:opacity 150ms ease-in-out}.module-auth.import--preview #user-import-preview[aria-busy=true] tbody{opacity:.4}.module-auth.import--preview #user-import-preview[aria-busy=true] .pagination a{pointer-events:none}.module-auth.import--preview th.field--line,.module-auth.import--preview td.field--line{width:60px;text-align:center} diff --git a/assets/js/admin.js b/assets/js/admin.js index ac26b93c..a49f283d 100644 --- a/assets/js/admin.js +++ b/assets/js/admin.js @@ -6,6 +6,7 @@ import AccountEdit from './components/AccountEdit.js'; import AccountMerge from './components/AccountMerge.js'; import Groups from './components/Groups.js'; import SearchUser from './components/SearchUser.js'; +import UserImport from './components/UserImport.js'; (function() { window.NAILS.ADMIN.registerPlugin( @@ -33,5 +34,10 @@ import SearchUser from './components/SearchUser.js'; 'SearchUser', new SearchUser(window.NAILS.ADMIN) ); + window.NAILS.ADMIN.registerPlugin( + 'nails/module-auth', + 'UserImport', + new UserImport(window.NAILS.ADMIN) + ); })(); diff --git a/assets/js/components/UserImport.js b/assets/js/components/UserImport.js new file mode 100644 index 00000000..4afbe942 --- /dev/null +++ b/assets/js/components/UserImport.js @@ -0,0 +1,1617 @@ +/** + * Drives the two pieces of the user import UI which need live data: the list of + * recent imports on the index, and the paged preview of an uploaded CSV. + * + * The admin bundle is loaded on every admin page, so each half no-ops unless its + * container is in the DOM. + */ +class UserImport { + + /** + * Construct UserImport + * @param {Object} adminController The admin controller + */ + constructor(adminController) { + + this.adminController = adminController; + + /** + * How often, in ms, to poll while something might change + * @type {Number} + */ + this.pollActive = 2500; + + /** + * How often, in ms, to poll when everything has settled + * @type {Number} + */ + this.pollIdle = 30000; + + /** + * How many page links either side of the current page the paginator + * shows; matches the `num_links` the admin pagination partial configures + * @type {Number} + */ + this.numLinks = 5; + + this.endpoint = window.SITE_URL + 'api/auth/import'; + + this.dom = { + list: document.getElementById('user-import-list'), + listBody: document.getElementById('user-import-list-body'), + preview: document.getElementById('user-import-preview'), + previewBody: document.getElementById('user-import-preview-body') + }; + + this.imports = []; + this.listTimeout = null; + + /** + * The shared modal, created on first use. Null means "not yet asked for", + * false means "asked for and unavailable". + * @type {Object|Boolean|null} + */ + this.modal = null; + + /** + * Null until the first page of the preview has loaded; this is what + * distinguishes the first load from a page change + * @type {Number|null} + */ + this.previewPage = null; + + /** + * The IDs of imports whose delete request is in flight, consulted when + * the actions cell renders so that the button's busy state survives a + * poll landing mid-request + * @type {Set} + */ + this.deleting = new Set(); + + /** + * Whether a confirmation is on screen. The modal is shared, so a second + * confirm() would reassign its onHide and leave the first promise + * unsettled. + * @type {Boolean} + */ + this.confirming = false; + + this.bindConfirmForms(); + this.bindDelete(); + this.bindDetails(); + + if (this.dom.list && this.dom.listBody) { + this.adminController.log('Constructing user import list'); + this.loadList(); + } + + if (this.dom.preview && this.dom.previewBody) { + this.adminController.log('Constructing user import preview'); + this.bindPreview(); + + /** + * When the CSV has gone the server has already said so in the table, + * and the API could only 404; asking would just stack a modal on top + * of the flash error which is already on the page. + */ + if (!this.dom.preview.dataset.csvMissing) { + this.loadPreview(1); + } + } + } + + // -------------------------------------------------------------------------- + + /** + * Fetches a URL and returns the decoded API envelope + * + * On failure the rejection carries the response's status, and the API's own + * message where it got far enough to render one: the poll only needs to + * know that something went wrong, but a delete the user asked for has to be + * able to say what. + * + * @param {String} url The URL to fetch + * @param {Object} options Any fetch options to apply, e.g. {method: 'DELETE'} + * @return {Promise} Resolves with the parsed response body + */ + request(url, options) { + + let settings = Object.assign({method: 'GET', credentials: 'same-origin'}, options || {}); + + // Merged over the caller's, so a caller cannot drop what the API needs + settings.headers = Object.assign({}, settings.headers, { + 'Accept': 'application/json', + 'X-Requested-With': 'XMLHttpRequest' + }); + + return fetch(url, settings) + .then((response) => response.text().then((body) => { + + let payload = null; + + /** + * In production an unhandled exception is re-thrown to the + * global error handler, so a 500 arrives as an HTML page rather + * than an envelope; the status is then all we have to go on. + */ + try { + payload = body ? JSON.parse(body) : null; + } catch (e) { + payload = null; + } + + if (!response.ok) { + let error = new Error( + (payload && payload.error) || `Request failed with status ${response.status}` + ); + error.status = response.status; + throw error; + } + + return payload || {}; + })); + } + + // -------------------------------------------------------------------------- + + /** + * Loads, and schedules the next load of, the list of recent imports + * @return {void} + */ + loadList() { + + /** + * The preview page runs the same class but has no list, and the delete + * handlers resync unconditionally; without this a failed delete there + * would reach renderList() and fall over on a null container. + */ + if (!this.dom.list || !this.dom.listBody) { + return; + } + + clearTimeout(this.listTimeout); + + this.request(this.endpoint) + .then((response) => { + + let imports = response.data || []; + + this.renderList(imports); + + // Poll quickly while something is in flight, slowly otherwise + let isBusy = imports.some((item) => { + return ['PENDING', 'VALIDATING', 'RUNNING'].indexOf(item.status) !== -1; + }); + + this.listTimeout = setTimeout(() => { + this.loadList(); + }, isBusy ? this.pollActive : this.pollIdle); + }) + .catch((error) => { + + /** + * This path polls every few seconds, so it must never raise a + * modal; one flaky response would otherwise become an unclosable + * stream of dialogs. The next poll will recover on its own. + */ + this.adminController.warn('Failed to load user imports', error); + + this.listTimeout = setTimeout(() => { + this.loadList(); + }, this.pollIdle); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Reconciles the rendered rows against the response + * @param {Array} imports The imports as returned by the API + * @return {void} + */ + renderList(imports) { + + let responseIds = imports.map((item) => item.id); + + for (let i = 0; i < imports.length; i++) { + let existing = this.imports.find((x) => x.id === imports[i].id); + if (existing) { + /** + * The stored item is what the delete confirmation reads its + * status and counts from, so it has to keep up; `dom` survives + * because the response never carries one. + */ + Object.assign(existing, imports[i]); + this.updateRow(existing.dom, imports[i]); + } else { + this.addRow(imports[i]); + } + } + + // Reverse loop so items can be spliced out as we go + for (let i = this.imports.length - 1; i >= 0; i--) { + if (responseIds.indexOf(this.imports[i].id) === -1) { + if (this.imports[i].dom && this.imports[i].dom.parentNode) { + this.imports[i].dom.parentNode.removeChild(this.imports[i].dom); + } + this.imports.splice(i, 1); + } + } + + /** + * Revealed on the first successful response and never hidden again, so an + * admin with no imports sees the empty state rather than nothing at all. + */ + this.dom.list.classList.remove('hidden'); + this.toggleListEmptyRow(); + } + + // -------------------------------------------------------------------------- + + /** + * Adds or removes the list's empty state row + * @return {void} + */ + toggleListEmptyRow() { + + let row = this.dom.listBody.querySelector('tr.user-import-list-empty'); + + if (this.imports.length) { + if (row) { + row.parentNode.removeChild(row); + } + return; + } + + if (!row) { + row = document.createElement('tr'); + row.classList.add('user-import-list-empty'); + row.appendChild(this.createCell( + 'no-data', + 'No imports found', + this.countColumns(this.dom.list) + )); + this.dom.listBody.appendChild(row); + } + } + + // -------------------------------------------------------------------------- + + /** + * Drops an import from the list, and from the reconciliation set + * + * Splicing `imports` is the point: leave the entry behind and the next + * poll's find() matches it, updateRow() writes into a detached node, the + * row never comes back, and the empty state never appears either. + * + * @param {Number} id The import to remove + * @return {void} + */ + removeImport(id) { + + let index = this.imports.findIndex((x) => x.id === id); + if (index === -1) { + return; + } + + let row = this.imports[index].dom; + if (row && row.parentNode) { + row.parentNode.removeChild(row); + } + + this.imports.splice(index, 1); + this.toggleListEmptyRow(); + } + + // -------------------------------------------------------------------------- + + /** + * Adds a row for an import which was not previously rendered + * @param {Object} item The import + * @return {void} + */ + addRow(item) { + + let tr = document.createElement('tr'); + + tr.appendChild(this.createCell(['field', 'field--id'], this.getIdCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--file'], this.getFileCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--status'], this.getStatusCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--progress'], this.getProgressCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--requested', 'datetime'], this.getRequestedCellHtml(item))); + tr.appendChild(this.createCell(['field', 'field--finished', 'datetime'], this.getFinishedCellHtml(item))); + + /** + * Left genuinely empty when there is nothing to do, so that admin's + * `td.actions:empty:before` supplies the "No Actions" label + */ + tr.appendChild(this.createCell('actions', this.getActionsCellHtml(item))); + + item.dom = tr; + this.imports.push(item); + this.dom.listBody.appendChild(tr); + } + + // -------------------------------------------------------------------------- + + /** + * Updates the cells of an already rendered import + * @param {Object} dom The row element + * @param {Object} item The import + * @return {void} + */ + updateRow(dom, item) { + + this.setCellHtml(dom, 'td.field--status', this.getStatusCellHtml(item)); + this.setCellHtml(dom, 'td.field--progress', this.getProgressCellHtml(item)); + this.setCellHtml(dom, 'td.field--finished', this.getFinishedCellHtml(item)); + this.setCellHtml(dom, 'td.actions', this.getActionsCellHtml(item)); + } + + // -------------------------------------------------------------------------- + + /** + * Sets a cell's contents, if the cell exists + * + * Guarded because this runs inside the poll loop; a renamed class should cost + * one stale cell, not a stream of errors every few seconds. + * + * @param {HTMLElement} row The row + * @param {String} selector The cell's selector + * @param {String} html The cell's contents + * @return {void} + */ + setCellHtml(row, selector, html) { + let cell = row.querySelector(selector); + if (cell) { + cell.innerHTML = html; + } + } + + // -------------------------------------------------------------------------- + + /** + * The ID cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getIdCellHtml(item) { + return this.escape(item.id); + } + + // -------------------------------------------------------------------------- + + /** + * The file cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getFileCellHtml(item) { + let name = item.source && item.source.filename ? item.source.filename : 'CSV'; + return `${this.escape(name)}`; + } + + // -------------------------------------------------------------------------- + + /** + * The status cell's contents + * + * The badge carries the colour; the cell itself is left untinted, so a row + * no longer changes colour wholesale. admin's `td small { display: block }` + * puts the error underneath without any help from us. + * + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getStatusCellHtml(item) { + + let html = `` + + this.escape(item.status) + + ''; + + /** + * The summary line only - the stored error is deliberately multi-line + * (see Processor::composeError()) and the whole thing here would turn + * every failed row into a wall of text. The Details button has the rest. + */ + if (item.error) { + html += `${this.escape(item.error_summary || item.error)}`; + } + + return html; + } + + // -------------------------------------------------------------------------- + + /** + * The badge modifier for an import's status + * + * Mapped rather than derived: the return goes straight into a `class` + * attribute, and a lookup is what guarantees it can only ever be one of the + * five modifiers the stylesheet declares - including for a status case + * shipped after this bundle was last built. + * + * @param {Object} item The import + * @return {String} The modifier class + */ + getStatusVariant(item) { + return { + DRAFT: 'badge--default', + PENDING: 'badge--info', + VALIDATING: 'badge--info', + RUNNING: 'badge--info', + COMPLETE: 'badge--success', + PARTIAL: 'badge--warning', + FAILED: 'badge--danger' + }[item.status] || 'badge--default'; + } + + // -------------------------------------------------------------------------- + + /** + * The progress cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getProgressCellHtml(item) { + + let progress = item.progress || {}; + let total = progress.row_count || 0; + + if (!total) { + return ''; + } + + let cursor = item.status === 'VALIDATING' + ? progress.validated_count + : progress.processed_count; + + let html = `${this.numberFormat(cursor)} of ${this.numberFormat(total)} (${progress.percent}%)`; + + /** + * Warnings get their own line rather than being folded into the errors: + * an errored row has no account and a warned one does, so they are not + * the same number to anybody reading this. + */ + if (progress.error_count) { + html += `${this.numberFormat(progress.error_count)} errored`; + } + + if (progress.warning_count) { + html += `${this.numberFormat(progress.warning_count)} warned`; + } + + return html; + } + + // -------------------------------------------------------------------------- + + /** + * The requested cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getRequestedCellHtml(item) { + let when = item.created ? item.created.formatted : ''; + let who = item.user ? item.user.name : null; + return who ? `${when}${this.escape(who)}` : when; + } + + // -------------------------------------------------------------------------- + + /** + * The finished cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getFinishedCellHtml(item) { + return item.finished + ? item.finished.formatted + : ''; + } + + // -------------------------------------------------------------------------- + + /** + * The actions cell's contents + * @param {Object} item The import + * @return {String} The cell's HTML + */ + getActionsCellHtml(item) { + + let buttons = []; + + if (item.status === 'DRAFT') { + buttons.push(`Review`); + } + + if (this.hasDetails(item)) { + buttons.push('`); + } + + if (item.log && item.log.url) { + buttons.push(`Log`); + } + + if (this.isDeletable(item)) { + /** + * Rendered busy while the request is in flight, because the poll + * re-renders this cell out from under the button - every 2.5s while + * something is active, and every 30s when nothing is, which is the + * case that actually bites. + */ + buttons.push(this.deleting.has(item.id) + ? '' + : '` + ); + } + + return buttons.join(' '); + } + + // -------------------------------------------------------------------------- + + /** + * Whether an import can be deleted + * + * Mirrors Status::isDeletable(). The server checks again and answers 409 if + * this is out of date, which it will be for up to one poll. + * + * @param {Object} item The import + * @return {Boolean} Whether it can be deleted + */ + isDeletable(item) { + return ['VALIDATING', 'RUNNING'].indexOf(item.status) === -1; + } + + // -------------------------------------------------------------------------- + + /** + * Whether an import has anything worth opening a modal for + * + * Either a stored reason, or per-row failures or warnings to list. A clean + * import has none of them and gets no button, which is what keeps the list + * uncluttered. + * + * @param {Object} item The import + * @return {Boolean} Whether there is detail to show + */ + hasDetails(item) { + let progress = item.progress || {}; + return Boolean(item.error) || + Boolean(progress.error_count) || + Boolean(progress.warning_count); + } + + // -------------------------------------------------------------------------- + + /** + * Wires up the preview's paging + * @return {void} + */ + bindPreview() { + + this.dom.previewPagingTop = this.dom.preview.querySelector('[data-paging="top"]'); + this.dom.previewPagingBottom = this.dom.preview.querySelector('[data-paging="bottom"]'); + + /** + * Delegated, because both paginators are re-rendered on every response; + * listeners bound to the buttons themselves would go stale immediately. + */ + this.dom.preview + .addEventListener('click', (event) => { + + let link = event.target.closest('a[data-page]'); + if (!link) { + return; + } + + event.preventDefault(); + + // Anchors cannot be disabled, so the busy state is the guard + if (this.dom.preview.getAttribute('aria-busy') === 'true') { + return; + } + + this.loadPreview(parseInt(link.dataset.page, 10), link); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Loads a page of the CSV preview + * @param {Number} page The page to load + * @param {HTMLElement} trigger The button which asked for it, if any + * @return {void} + */ + loadPreview(page, trigger) { + + let id = this.dom.preview.dataset.importId; + let isFirstLoad = this.previewPage === null; + + /** + * On the first load the server has already rendered a spinner row, so + * there is nothing to dim and no button to put a spinner on. + */ + if (!isFirstLoad) { + this.setPreviewBusy(true, trigger); + } + + this.request(`${this.endpoint}/${id}/rows?page=${page}`) + .then((response) => { + + this.previewPage = page; + this.renderPreview(response.data || [], response.meta || {}); + + // The first load is already at the top of the page; only a page change moves + if (!isFirstLoad) { + this.scrollToPreview(); + } + }) + .catch((error) => { + + this.adminController.warn('Failed to load user import preview', error); + + /** + * On a page change the rows on screen are still valid, so they + * stay; losing what you were reading because page seven timed out + * is worse than the error itself. + */ + if (isFirstLoad) { + this.renderPreviewMessage('The preview could not be loaded'); + } + + this.alert( + 'The preview could not be loaded', + 'Something went wrong fetching this page of the CSV. Please reload the page to try again.' + ); + }) + .finally(() => { + this.setPreviewBusy(false); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Toggles the preview's busy state + * + * `aria-busy` doubles as the styling hook, so no class has to be invented; it + * is also what stops the paging links being followed, since an anchor has no + * disabled state. + * + * @param {Boolean} isBusy Whether a request is in flight + * @param {HTMLElement} trigger The link which asked for it, if any + * @return {void} + */ + setPreviewBusy(isBusy, trigger) { + + this.dom.preview.setAttribute('aria-busy', isBusy ? 'true' : 'false'); + + if (isBusy) { + + if (trigger) { + this.previewTrigger = {el: trigger, label: trigger.innerHTML}; + trigger.innerHTML = ''; + } + + } else if (this.previewTrigger) { + + /** + * A successful load re-renders the paginator, taking the link with + * it; the link is only still around when the request failed, and then + * it must not be left spinning. + */ + if (this.previewTrigger.el.isConnected) { + this.previewTrigger.el.innerHTML = this.previewTrigger.label; + } + + this.previewTrigger = null; + } + } + + // -------------------------------------------------------------------------- + + /** + * Renders a page of the CSV preview + * @param {Array} rows The rows to render + * @param {Object} meta The response's meta data + * @return {void} + */ + renderPreview(rows, meta) { + + let header = meta.header || []; + let pagination = meta.pagination || {}; + + if (!rows.length) { + this.renderPreviewMessage('There is nothing to preview'); + + } else { + + this.dom.previewBody.innerHTML = ''; + + for (let i = 0; i < rows.length; i++) { + + let tr = document.createElement('tr'); + tr.appendChild(this.createCell(['field', 'field--line'], String(rows[i].line))); + + for (let j = 0; j < header.length; j++) { + let value = rows[i].data[header[j]]; + tr.appendChild(this.createCell( + this.fieldClass(header[j]).split(' '), + value === null || value === '' || typeof value === 'undefined' + ? '' + : this.escape(String(value)) + )); + } + + this.dom.previewBody.appendChild(tr); + } + } + + this.renderPaginator(this.dom.previewPagingTop, pagination); + this.renderPaginator(this.dom.previewPagingBottom, pagination); + } + + // -------------------------------------------------------------------------- + + /** + * Replaces the preview's rows with a single full width message + * @param {String} message The message to show + * @return {void} + */ + renderPreviewMessage(message) { + + let tr = document.createElement('tr'); + tr.appendChild(this.createCell( + 'no-data', + this.escape(message), + this.countColumns(this.dom.preview) + )); + + this.dom.previewBody.innerHTML = ''; + this.dom.previewBody.appendChild(tr); + } + + // -------------------------------------------------------------------------- + + /** + * Renders a paginator into a container + * + * This mirrors the markup produced by module-admin's + * `admin/views/_components/pagination.php`, which is the source of truth, with + * the same `num_links: 5` / `use_page_numbers: true` configuration. The + * partial cannot be used directly because paging happens over the API with no + * page load, so the whole paginator is redrawn from each response instead. + * + * The links must be anchors: admin styles them as + * `.pagination ul li.page a`, so a button would come out unstyled. + * + * @param {HTMLElement} container The element to render into + * @param {Object} pagination The `meta.pagination` object from the API + * @return {void} + */ + renderPaginator(container, pagination) { + + if (!container) { + return; + } + + let page = pagination.page || 1; + let perPage = pagination.per_page || 0; + let total = pagination.total || 0; + let pages = perPage ? Math.ceil(total / perPage) : 0; + + // "Records X to Y of Z", arithmetic lifted from the partial + let start = (page * perPage) - perPage; + let end = start + perPage; + if (total > 0) { + start++; + } + if (end > total) { + end = total; + } + + let items = []; + + // Like CodeIgniter, render no links at all for an empty or single page + if (pages > 1) { + + if (page > this.numLinks + 1) { + items.push(this.paginatorItem('First', 1, 'first')); + } + + if (page !== 1) { + items.push(this.paginatorItem('‹', page - 1, 'previous')); + } + + let from = Math.max(1, page - this.numLinks); + let to = Math.min(pages, page + this.numLinks); + + for (let i = from; i <= to; i++) { + items.push(i === page + ? `
  • ${i}
  • ` + : this.paginatorItem(String(i), i)); + } + + if (page < pages) { + items.push(this.paginatorItem('›', page + 1, 'next')); + } + + if (page + this.numLinks < pages) { + items.push(this.paginatorItem('Last', pages, 'last')); + } + } + + container.innerHTML = [ + '' + ].join(''); + } + + // -------------------------------------------------------------------------- + + /** + * A single clickable paginator item + * @param {String} label The item's label; may contain entities + * @param {Number} page The page the item navigates to + * @param {String} modifier An extra class for the item, if any + * @return {String} The item's HTML + */ + paginatorItem(label, page, modifier) { + return `
  • ` + + `${label}` + + '
  • '; + } + + // -------------------------------------------------------------------------- + + /** + * Brings the top of the preview back into view + * + * Paging from the bottom paginator replaces the rows above the viewport, so + * without this the page you just asked for is off-screen and has to be + * scrolled to by hand. + * + * @return {void} + */ + scrollToPreview() { + + // Measured rather than hard coded; admin's header is fixed and overlaps + let header = document.querySelector('body > .header'); + let offset = (header ? header.offsetHeight : 0) + 8; + + let top = this.dom.preview.getBoundingClientRect().top + window.scrollY - offset; + + let reduceMotion = window.matchMedia + && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + + window.scrollTo({ + top: Math.max(0, top), + behavior: reduceMotion ? 'auto' : 'smooth' + }); + } + + // -------------------------------------------------------------------------- + + /** + * Creates a table cell + * @param {String|Array} classes The classes to apply + * @param {String} html The cell's contents + * @param {Number} colspan The number of columns to span, if any + * @return {Object} The cell element + */ + createCell(classes, html, colspan) { + + let cell = document.createElement('td'); + + if (Array.isArray(classes)) { + cell.classList.add(...classes); + } else { + cell.classList.add(classes); + } + + if (colspan) { + cell.setAttribute('colspan', colspan); + } + + if (html) { + cell.innerHTML = html; + } + + return cell; + } + + // -------------------------------------------------------------------------- + + /** + * The number of columns in a container's table + * @param {HTMLElement} container The element containing the table + * @return {Number} The column count + */ + countColumns(container) { + return container.querySelectorAll('thead th').length; + } + + // -------------------------------------------------------------------------- + + /** + * Normalises a column label into a cell class, as module-admin does + * @param {String} label The column's label + * @return {String} The cell's classes + */ + fieldClass(label) { + return 'field field--' + String(label) + .toLowerCase() + .replace(/[^a-z0-9 \-_]/g, '') + .replace(/[ _]/g, '-'); + } + + // -------------------------------------------------------------------------- + + /** + * Formats a number as PHP's number_format() does + * + * Deliberately not toLocaleString(); number_format() is not locale aware, so + * this is what actually matches the markup we are mirroring. + * + * @param {Number} value The value to format + * @return {String} The formatted value + */ + numberFormat(value) { + return String(value || 0).replace(/\B(?=(\d{3})+(?!\d))/g, ','); + } + + // -------------------------------------------------------------------------- + + /** + * Returns the shared modal, creating it on first use + * @return {Object|null} The modal instance, or null if admin's Modal is unavailable + */ + getModal() { + + if (this.modal === null) { + try { + this.modal = this.adminController.getInstance('Modal', 'nails/module-admin').create(); + } catch (e) { + this.adminController.warn('Admin\'s Modal component is unavailable', e); + this.modal = false; + } + } + + return this.modal || null; + } + + // -------------------------------------------------------------------------- + + /** + * Shows a message in a modal + * @param {String} title The modal's title + * @param {String} message The message to show + * @return {void} + */ + alert(title, message) { + + let modal = this.getModal(); + + if (!modal) { + window.alert(`${title}\n\n${message}`); + return; + } + + let p = document.createElement('p'); + p.innerText = message; + + // Clear anything a previous confirm() left on the shared instance + modal.onHide(() => { + }); + + modal + .setTitle(this.escape(title)) + .setBody(p) + .clearActions() + .addAction('Close', ['btn-primary'], (event, instance) => instance.hide()) + .show(); + } + + // -------------------------------------------------------------------------- + + /** + * Asks the user to confirm an action + * @param {String} title The modal's title + * @param {String} message The question to ask + * @param {String} action The confirming action's label + * @param {Boolean} warn Whether to present the question as a warning + * @return {Promise} Resolves with whether the action was confirmed + */ + confirm(title, message, action, warn) { + + return new Promise((resolve) => { + + let modal = this.getModal(); + + if (!modal) { + // Better an ugly confirmation than a button which silently does nothing + resolve(window.confirm(message)); + return; + } + + let settled = false; + let settle = (confirmed) => { + if (settled) { + return; + } + settled = true; + resolve(confirmed); + }; + + let p = document.createElement('p'); + p.innerText = message; + + let body = p; + + if (warn) { + + /** + * Wrapped in an alert so the question reads as a warning at a + * glance, rather than as the same paragraph every other + * confirmation shows. Following Notes::showError() in + * module-admin, which hands setBody() an `.alert` of its own - + * the message still goes in through innerText, the alert is the + * only thing being added. + */ + body = document.createElement('div'); + body.classList.add('alert', 'alert-warning'); + + /** + * Inline rather than a rule in this module's stylesheet: the + * admin bundle is global, so `.modal__body > .alert` would + * reach every other admin modal too - module-admin's own alerts + * included - to spare this one the 20px it does not need. + */ + body.style.marginBottom = '0'; + + body.appendChild(p); + } + + // Escape to close is built into the modal, and counts as a cancellation + modal.onHide(() => settle(false)); + + modal + .setTitle(this.escape(title)) + .setBody(body) + .clearActions() + .addAction(this.escape(action), ['btn-primary'], (event, instance) => { + settle(true); + instance.hide(); + }) + .addAction('Cancel', ['btn-danger'], (event, instance) => { + settle(false); + instance.hide(); + }) + .show(); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Routes the submission of flagged forms through a modal confirmation + * + * The listener is on the form rather than the button so that the Delete + * button, which is adopted into the floating controls by the `form` + * attribute, keeps working. + * + * @return {void} + */ + bindConfirmForms() { + + document + .querySelectorAll('form.js-user-import-confirm') + .forEach((form) => { + form.addEventListener('submit', (event) => { + + event.preventDefault(); + + /** + * Held for the whole chain, not just one modal: the shared + * instance has a single onHide slot, so a Delete + * confirmation opening over this one would leave this + * promise unsettled. + */ + this.confirming = true; + + this + .confirmForm(form) + .then((confirmed) => { + if (confirmed) { + /** + * Called off the prototype in case a control is + * ever named "submit", which would shadow it. + * Note this does not fire the submit event, so it + * cannot loop back into this handler. + */ + HTMLFormElement.prototype.submit.call(form); + } + }) + .finally(() => { + this.confirming = false; + }); + }); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Confirms a form's submission, warning about it first if it needs it + * + * A form which names a `data-warn-field` checkbox, and has it unticked, + * gets a warning before the confirmation; Continue is an implied tick, and + * chains straight on into that confirmation. Anything else - no such field, + * or a box already ticked - goes straight there, exactly as every other + * flagged form does. + * + * The two run in sequence over the one shared modal, which keeps them clear + * of module-admin's stacking, and there is nothing to see in between: the + * hide and the re-show both land in the same task. + * + * @param {HTMLFormElement} form The form being submitted + * @return {Promise} Resolves with whether the submission was confirmed + */ + confirmForm(form) { + + let checkbox = this.getWarnCheckbox(form); + + // A local, so the regular confirmation is written once for both routes + let confirmSubmit = () => this.confirm( + form.dataset.confirmTitle || 'Are you sure?', + form.dataset.confirmBody || 'Please confirm you\'d like to continue with this action.', + form.dataset.confirmAction || 'OK' + ); + + if (!checkbox || checkbox.checked) { + return confirmSubmit(); + } + + return this + .confirm( + form.dataset.warnTitle || 'Are you sure?', + form.dataset.warnBody || 'Please confirm you\'d like to continue with this action.', + form.dataset.warnAction || 'Continue', + true + ) + .then((acknowledged) => { + + if (!acknowledged) { + return false; + } + + /** + * The real checkbox, rather than a hidden input, so the value + * reaches the server exactly as a tick would and the page shows + * what was agreed to. It stays ticked if the confirmation is + * then cancelled - the decision was made, and a second Import + * click should not have to make it again. + */ + checkbox.checked = true; + + return confirmSubmit(); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Resolves the checkbox a form wants warning about, if it names one + * + * Looked up through `form.elements` rather than interpolated into a + * selector, so a server-authored name never becomes part of one. + * + * @param {HTMLFormElement} form The form being submitted + * @return {HTMLElement|null} The checkbox, or null if the form has no such field + */ + getWarnCheckbox(form) { + + let name = form.dataset.warnField; + + if (!name) { + return null; + } + + let field = form.elements[name]; + + // A repeated name comes back as a RadioNodeList, which has no `type` + return field && field.type === 'checkbox' ? field : null; + } + + // -------------------------------------------------------------------------- + + /** + * Wires up the Details buttons on both the list and the preview + * + * Delegated from the document for the same two reasons bindDelete() is; see + * the note there. + * + * @return {void} + */ + bindDetails() { + + document.addEventListener('click', (event) => { + + let button = event.target.closest + ? event.target.closest('button.js-user-import-details') + : null; + + if (button) { + event.preventDefault(); + this.showDetails(parseInt(button.dataset.importId, 10)); + } + }); + } + + // -------------------------------------------------------------------------- + + /** + * Shows what went wrong with an import + * + * The modal opens straight away with a spinner rather than after the + * requests land, so a click always does something visible; the body is + * replaced in place once the detail arrives. + * + * @param {Number} id The import's ID + * @return {void} + */ + showDetails(id) { + + let modal = this.getModal(); + let cached = this.imports.find((item) => item.id === id); + + if (!modal) { + window.alert(cached && cached.error + ? cached.error + : 'The details for this import are unavailable.'); + return; + } + + let loading = document.createElement('p'); + let spinner = document.createElement('span'); + spinner.className = 'user-import-spinner'; + spinner.setAttribute('aria-hidden', 'true'); + loading.appendChild(spinner); + loading.appendChild(document.createTextNode(' Loading the details\u2026')); + + // Clear anything a previous confirm() left on the shared instance + modal.onHide(() => { + }); + + modal + .setTitle(`Import #${id}`) + .setBody(loading) + .clearActions() + .addAction('Close', ['btn-primary'], (event, instance) => instance.hide()) + .show(); + + Promise + .all([ + this.request(`${this.endpoint}/${id}`), + this.request(`${this.endpoint}/${id}/items?status=ERROR,WARNING`) + ]) + .then(([job, items]) => { + + let item = job.data || {}; + + modal + .setBody(this.buildDetailsBody(item, items.data || [], items.meta || {})) + .clearActions(); + + if (item.log && item.log.url) { + modal.addAction('Download the full log', ['btn-default'], () => { + window.location = item.log.url; + }); + } + + modal.addAction('Close', ['btn-primary'], (event, instance) => instance.hide()); + }) + .catch((error) => { + + this.adminController.warn('Failed to load user import details', error); + + let p = document.createElement('p'); + p.innerText = error.message || 'The details for this import could not be loaded.'; + + modal.setBody(p); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Builds the details modal's body + * + * Everything goes in via textContent rather than innerHTML. Both blocks + * carry values lifted straight out of the uploaded CSV, so there is no + * escaping decision to get wrong here. + * + * @param {Object} item The import, as returned by the API + * @param {Array} rows The import's failing and warned rows + * @param {Object} meta The rows response's meta + * @return {DocumentFragment} The body + */ + buildDetailsBody(item, rows, meta) { + + let fragment = document.createDocumentFragment(); + + if (item.error) { + let log = document.createElement('pre'); + log.className = 'user-import-details__log'; + log.textContent = item.error; + fragment.appendChild(log); + } + + if (rows.length) { + + let total = (meta.pagination || {}).total || rows.length; + + if (total > rows.length) { + let note = document.createElement('p'); + note.className = 'text-muted'; + note.textContent = 'Showing the first ' + this.numberFormat(rows.length) + + ' of ' + this.numberFormat(total) + ' affected rows; the log has every one.'; + fragment.appendChild(note); + } + + /** + * Each line carries its status, because the two the request asks for + * do not mean the same thing: an ERROR row has no account, a WARNING + * row has one which needs finishing by hand. + */ + let list = document.createElement('pre'); + list.className = 'user-import-details__rows'; + list.textContent = rows + .map((row) => `Line ${row.line} [${row.status}]: ` + + `${row.message || '(no reason was recorded)'}`) + .join('\n'); + fragment.appendChild(list); + } + + /** + * A bad header or a missing CSV fails before a single row is recorded, + * so an empty modal is a real outcome and needs saying out loud. + */ + if (!fragment.childNodes.length) { + let p = document.createElement('p'); + p.textContent = 'No further detail was recorded for this import.'; + fragment.appendChild(p); + } + + return fragment; + } + + // -------------------------------------------------------------------------- + + /** + * Wires up the Delete buttons on both the list and the preview + * + * Delegated from the document rather than from a container, for two + * reasons: the list's buttons are re-rendered on every poll, and the + * preview's button is rendered by admin's floating controls, which sit + * outside `#user-import-preview` - so a listener scoped the way + * bindPreview() scopes its own would never see it. The class is specific + * enough that this costs nothing on the admin pages which have neither. + * + * @return {void} + */ + bindDelete() { + + document.addEventListener('click', (event) => { + + let button = event.target.closest + ? event.target.closest('button.js-user-import-delete') + : null; + + if (button) { + event.preventDefault(); + this.deleteImport(button); + } + }); + } + + // -------------------------------------------------------------------------- + + /** + * Deletes an import, once the user has confirmed it + * + * @param {HTMLElement} button The Delete button which was clicked + * @return {void} + */ + deleteImport(button) { + + let id = parseInt(button.dataset.importId, 10); + + // A confirmation is already up, or this one is already on its way out + if (this.confirming || this.deleting.has(id)) { + return; + } + + /** + * The list knows the item; the preview page does not, and hands the two + * things the copy turns on over as attributes instead. + */ + let item = this.imports.find((x) => x.id === id); + let status = item ? item.status : button.dataset.status; + + // A warned row has an account too, so it is one of the accounts which stays + let created = item + ? (((item.progress || {}).success_count || 0) + ((item.progress || {}).warning_count || 0)) + : parseInt(button.dataset.createdCount || '0', 10); + + let label = button.innerHTML; + + this.confirming = true; + + this + .confirm( + 'Delete this import?', + this.getDeleteConfirmBody(status, created), + 'Delete' + ) + .then((confirmed) => { + + this.confirming = false; + + if (!confirmed) { + return; + } + + /** + * The clicked button is styled here for immediate feedback, and + * the id is recorded so that getActionsCellHtml() reproduces + * that state if the poll re-renders the cell mid-request. + */ + this.deleting.add(id); + button.disabled = true; + button.innerHTML = ''; + + return this + .request(button.dataset.deleteUrl, {method: 'DELETE'}) + .then(() => this.onDeleted(id, button)) + .catch((error) => this.onDeleteFailed(id, error)) + .finally(() => { + + /** + * In `finally`, never in `then`: an id left in the set + * by a failed request would disable that row's button + * on every subsequent poll, with no way back but a + * reload. + */ + this.deleting.delete(id); + + // Only still here if the delete failed; success took the row, or the page + if (button.isConnected) { + button.disabled = false; + button.innerHTML = label; + } + }); + }); + } + + // -------------------------------------------------------------------------- + + /** + * Drops a deleted import, and resyncs + * + * The row goes now rather than on the next poll; at pollIdle that would be + * half a minute of a row which is already gone. loadList() then reconciles + * against the server, and clears the pending timer as it goes. + * + * @param {Number} id The import which was deleted + * @param {HTMLElement} button The button which asked + * @return {void} + */ + onDeleted(id, button) { + + this.removeImport(id); + + // The preview is a page about something which no longer exists + if (button.dataset.redirect) { + window.location = button.dataset.redirect; + return; + } + + this.loadList(); + } + + // -------------------------------------------------------------------------- + + /** + * Reports a delete which did not happen + * + * loadList()'s refusal to raise a modal does not apply here: that is about + * not stacking dialogs every few seconds, and this is a one-off the user + * asked for. + * + * @param {Number} id The import which was not deleted + * @param {Error} error The rejection, carrying .status where we got that far + * @return {void} + */ + onDeleteFailed(id, error) { + + this.adminController.warn('Failed to delete user import', error); + + /** + * Already gone - most likely deleted in another tab. There is nothing + * to apologise for; the row disappearing is the whole message. + */ + if (error.status === 404) { + this.removeImport(id); + this.loadList(); + return; + } + + this.alert( + 'The import could not be deleted', + error.status === 409 + ? 'This import has started running since the page was loaded, so it can no longer be deleted.' + : error.message + ); + + // Whatever the server thinks, the list should agree with it + this.loadList(); + } + + // -------------------------------------------------------------------------- + + /** + * The body of the delete confirmation + * + * What matters is not the status but whether accounts exist: deleting the + * job never deletes the users it created, and a FAILED job may well have + * created some before it stopped - `user_import_item.user_id` is + * ON DELETE SET NULL precisely so that it cannot reach them. Saying + * otherwise invites an admin to read "Delete" as "undo". + * + * @param {String} status The import's status + * @param {Number} created How many accounts it created + * @return {String} The message + */ + getDeleteConfirmBody(status, created) { + + if (created) { + return 'The uploaded CSV and its log will be deleted. The ' + + `${this.numberFormat(created)} user ` + + `${created === 1 ? 'account' : 'accounts'} this import created ` + + 'will NOT be deleted. This cannot be undone.'; + } + + if (status === 'DRAFT' || status === 'PENDING') { + return 'The uploaded CSV will be deleted and no users will be created. This cannot be undone.'; + } + + return 'The uploaded CSV and its log will be deleted. ' + + 'This import created no user accounts. This cannot be undone.'; + } + + // -------------------------------------------------------------------------- + + /** + * Escapes a value for insertion into the DOM + * @param {String} value The value to escape + * @return {String} The escaped value + */ + escape(value) { + let div = document.createElement('div'); + div.appendChild(document.createTextNode(value)); + return div.innerHTML; + } +} + +export default UserImport; diff --git a/assets/sass/admin.scss b/assets/sass/admin.scss index 621a5bc3..1e2b4d63 100644 --- a/assets/sass/admin.scss +++ b/assets/sass/admin.scss @@ -278,3 +278,147 @@ } } } + +/** + * Contextual badges. module-admin ships a bare `.badge` and deliberately no + * variants - it does not import Bootstrap's `_labels.scss`, and `.label` there + * is a form-label selector (`float: left; width: 130px`), so it must not be + * pressed into service as a pill. + * + * Even the neutral case needs an explicit colour: admin's own badge background + * is `rgba(0, 0, 0, 0.25)`, which over a white cell resolves to #bfbfbf - 1.8:1 + * against the white text Bootstrap gives it, a flat contrast failure. + * + * The hexes are admin's `$color-*-text` palette written out longhand, because + * this stylesheet is standalone and cannot reach its variables. All five clear + * 4.5:1 against white at the 12px bold Bootstrap's base rule sets, which also + * supplies the colour, weight and radius. Doubled up on `.badge` so the rules + * do not depend on which stylesheet loads last. + */ +.badge { + &.badge--default { + background-color: #555555; // 7.5:1 + } + + &.badge--info { + background-color: #31708f; // 5.5:1 + } + + &.badge--success { + background-color: #3c763d; // 5.5:1 + } + + &.badge--warning { + background-color: #8a6d3b; // 4.9:1 + } + + &.badge--danger { + background-color: #a94442; // 5.9:1 + } +} + +/** + * A spinner of our own. The admin theme has no reusable one - its only spinner + * is scoped to `.search .mask` - and Bootstrap's `.spinner-border` is not in the + * installed build, so this is self contained. + */ +@keyframes user-import-spin { + to { + transform: rotate(360deg); + } +} + +.user-import-spinner { + display: inline-block; + width: 0.85em; + height: 0.85em; + vertical-align: -0.1em; + border: 2px solid currentColor; + border-right-color: transparent; + border-radius: 50%; + animation: user-import-spin 0.75s linear infinite; +} + +/** + * The details modal's body. Top level, alongside the spinner, because admin's + * Modal appends its container to `document.body` - outside `.module-auth` - so + * a rule nested in there would never match. + */ +.user-import-details { + p { + margin: 0 0 0.5rem; + } + + /** + * The modal body already scrolls (`max-height: 80vh` in module-admin's + * modal.scss), so each block is capped and owns its own scroll; a scroller + * nested inside a scroller makes both awkward to use. + * + * `pre-wrap` is what keeps a long "already registered" message wrapping + * instead of forcing the whole modal sideways. + */ + &__log, + &__rows { + max-height: 40vh; + overflow: auto; + margin: 0 0 0.75rem; + padding: 0.75rem; + font-size: 12px; + line-height: 1.5; + white-space: pre-wrap; + word-break: break-word; + background-color: #f5f5f5; + border: 1px solid #d9d9d9; + border-radius: 4px; + } +} + +.module-auth { + &.import { + /** + * Four btn-xs now that Details is there alongside Review, Log and + * Delete; the 225px this used to carry wrapped them. + */ + th.actions, + td.actions { + width: 275px; + min-width: 275px; + } + } + + &.import--preview { + #user-import-preview { + /** + * Fade the outgoing rows while the next page is in flight. module-admin + * has no busy convention to borrow - the `.search > .mask` spinner's + * `search-spin` keyframes are never declared, so it does not animate - + * so this is the one rule we own. `aria-busy` doubles as the hook, so + * no class has to be invented. + */ + tbody { + transition: opacity 150ms ease-in-out; + } + + &[aria-busy='true'] { + tbody { + opacity: 0.4; + } + + // Anchors have no disabled state, so stop the clicks here too + .pagination a { + pointer-events: none; + } + } + } + + /** + * The line number is a gutter, not data; without a width it takes an equal + * share of the table alongside the CSV's own columns. + */ + th.field--line, + td.field--line { + width: 60px; + text-align: center; + } + } +} diff --git a/auth/config/email_types.php b/auth/config/email_types.php index 4a4d41a4..70acd241 100644 --- a/auth/config/email_types.php +++ b/auth/config/email_types.php @@ -46,6 +46,28 @@ 'can_unsubscribe' => false, 'factory' => Constants::MODULE_SLUG . '::EmailPasswordUpdated', ], + (object) [ + 'slug' => 'user_import_complete', + 'name' => 'Auth: User Import Complete', + 'description' => 'Email sent to the administrator who requested a user import when it finishes.', + 'template_header' => '', + 'template_body' => 'auth/email/user_import_complete', + 'template_footer' => '', + 'default_subject' => 'Your user import has finished', + 'can_unsubscribe' => false, + 'factory' => Constants::MODULE_SLUG . '::EmailUserImportComplete', + ], + (object) [ + 'slug' => 'user_import_failed', + 'name' => 'Auth: User Import Failed', + 'description' => 'Email sent to the administrator who requested a user import when it is rejected.', + 'template_header' => '', + 'template_body' => 'auth/email/user_import_failed', + 'template_footer' => '', + 'default_subject' => 'Your user import was rejected', + 'can_unsubscribe' => false, + 'factory' => Constants::MODULE_SLUG . '::EmailUserImportFailed', + ], (object) [ 'slug' => 'verify_email', 'name' => 'Auth: Verify Email (Generic)', diff --git a/auth/controllers/Login.php b/auth/controllers/Login.php index 371db08c..225e53e2 100644 --- a/auth/controllers/Login.php +++ b/auth/controllers/Login.php @@ -22,6 +22,8 @@ use Nails\Auth\Resource; use Nails\Auth\Service\Authentication; use Nails\Auth\Service\SocialSignOn; +use Nails\Auth\Validator\User\Identifier; +use Nails\Auth\Validator\User\Identity; use Nails\Cdn\Service\Cdn; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\ValidationException; @@ -113,8 +115,6 @@ public function index() $oInput = Factory::service('Input'); /** @var \App\Auth\Model\User $oUserModel */ $oUserModel = Factory::model('User', Constants::MODULE_SLUG); - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); /** @var Authentication $oAuthService */ $oAuthService = Factory::service('Authentication', Constants::MODULE_SLUG); /** @var SocialSignOn $oSocial */ @@ -128,20 +128,9 @@ public function index() try { - $oFormValidation - ->buildValidator([ - 'identifier' => array_values(array_filter([ - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'EMAIL' ? [ - $oFormValidation::RULE_REQUIRED, - $oFormValidation::RULE_VALID_EMAIL, - ] : null, - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'USERNAME' ? [$oFormValidation::RULE_REQUIRED] : null, - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'BOTH' ? [$oFormValidation::RULE_REQUIRED] : null, - ]))[0], - 'password' => [$oFormValidation::RULE_REQUIRED], - 'remember' => [], - ]) - ->run(); + (new Identifier()) + ->addRules(['password' => [FormValidation::RULE_REQUIRED]]) + ->run($oInput->post()); if (appSetting('user_login_captcha_enabled', 'auth')) { if (!$oCaptchaService->verify()) { @@ -945,66 +934,53 @@ protected function socialSignOnRequestData(array &$aRequiredData): void $oInput = Factory::service('Input'); if ($oInput->post()) { - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); + // Only capture the identity fields we've been asked for + $aRules = []; - if (isset($aRequiredData['email'])) { - $oFormValidation->set_rules('email', 'email', 'trim|required|valid_email|is_unique[' . \Nails\Config::get('NAILS_DB_PREFIX') . 'user_email.email]'); + if (!isset($aRequiredData['email'])) { + $aRules['email'] = []; } - if (isset($aRequiredData['username'])) { - $oFormValidation->set_rules('username', 'username', 'trim|required|is_unique[' . \Nails\Config::get('NAILS_DB_PREFIX') . 'user.username]'); + if (!isset($aRequiredData['username'])) { + $aRules['username'] = []; } if (empty($aRequiredData['first_name'])) { - $oFormValidation->set_rules('first_name', '', 'trim|required'); + $aRules['first_name'] = ['trim', FormValidation::RULE_REQUIRED]; } if (empty($aRequiredData['last_name'])) { - $oFormValidation->set_rules('last_name', '', 'trim|required'); + $aRules['last_name'] = ['trim', FormValidation::RULE_REQUIRED]; } - $oFormValidation->set_message('required', lang('fv_required')); - $oFormValidation->set_message('valid_email', lang('fv_valid_email')); + try { - if (\Nails\Config::get('APP_NATIVE_LOGIN_USING') == 'EMAIL') { - $oFormValidation->set_message( - 'is_unique', - lang('fv_email_already_registered', siteUrl('auth/password/forgotten')) - ); - } elseif (\Nails\Config::get('APP_NATIVE_LOGIN_USING') == 'USERNAME') { - $oFormValidation->set_message( - 'is_unique', - lang('fv_username_already_registered', siteUrl('auth/password/forgotten')) - ); - } else { - $oFormValidation->set_message( - 'is_unique', - lang('fv_identity_already_registered', siteUrl('auth/password/forgotten')) - ); - } + $oValidator = (new Identity()) + ->setRules($aRules) + ->setLabels(['email' => 'email', 'username' => 'username']) + ->run($oInput->post()); - if ($oFormValidation->run()) { + // Valid! Ensure required data is set correctly then allow system to move on. + $aPost = $oValidator->getValidatedData(); - // Valid!Ensure required data is set correctly then allow system to move on. if (isset($aRequiredData['email'])) { - $aRequiredData['email'] = $oInput->post('email'); + $aRequiredData['email'] = $aPost['email'] ?? null; } if (isset($aRequiredData['username'])) { - $aRequiredData['username'] = $oInput->post('username'); + $aRequiredData['username'] = $aPost['username'] ?? null; } if (empty($aRequiredData['first_name'])) { - $aRequiredData['first_name'] = $oInput->post('first_name'); + $aRequiredData['first_name'] = $aPost['first_name'] ?? null; } if (empty($aRequiredData['last_name'])) { - $aRequiredData['last_name'] = $oInput->post('last_name'); + $aRequiredData['last_name'] = $aPost['last_name'] ?? null; } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); $this->socialSignOnRequestDataForm($aRequiredData, $provider); } diff --git a/auth/controllers/MfaDevice.php b/auth/controllers/MfaDevice.php index e893123b..68fb126d 100644 --- a/auth/controllers/MfaDevice.php +++ b/auth/controllers/MfaDevice.php @@ -11,6 +11,7 @@ */ use Nails\Common\Exception\FactoryException; +use Nails\Common\Exception\ValidationException; use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; use Nails\Factory; @@ -82,12 +83,14 @@ protected function setupDevice() /** @var FormValidation $oFormValidation */ $oFormValidation = Factory::service('FormValidation'); - $oFormValidation->set_rules('mfa_secret', '', 'required'); - $oFormValidation->set_rules('mfa_code', '', 'required'); + try { - $oFormValidation->set_message('required', lang('fv_required')); - - if ($oFormValidation->run()) { + $oFormValidation + ->buildValidator([ + 'mfa_secret' => [FormValidation::RULE_REQUIRED], + 'mfa_code' => [FormValidation::RULE_REQUIRED], + ]) + ->run(); $sSecret = $oInput->post('mfa_secret'); $sMfaCode = $oInput->post('mfa_code'); @@ -108,8 +111,8 @@ protected function setupDevice() $this->oUserFeedback->error('Sorry, that code failed to validate. Please try again.'); } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); } } @@ -152,10 +155,13 @@ protected function requestCode() /** @var FormValidation $oFormValidation */ $oFormValidation = Factory::service('FormValidation'); - $oFormValidation->set_rules('mfa_code', '', 'required'); - $oFormValidation->set_message('required', lang('fv_required')); + try { - if ($oFormValidation->run()) { + $oFormValidation + ->buildValidator([ + 'mfa_code' => [FormValidation::RULE_REQUIRED], + ]) + ->run(); /** @var Authentication $oAuthService */ $oAuthService = Factory::service('Authentication', Constants::MODULE_SLUG); @@ -171,8 +177,8 @@ protected function requestCode() )); } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); } } diff --git a/auth/controllers/MfaQuestion.php b/auth/controllers/MfaQuestion.php index 57255f67..d827e66d 100644 --- a/auth/controllers/MfaQuestion.php +++ b/auth/controllers/MfaQuestion.php @@ -15,6 +15,7 @@ use Nails\Auth\Service\Authentication; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\NailsException; +use Nails\Common\Exception\ValidationException; use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; use Nails\Factory; @@ -112,44 +113,35 @@ public function index() /** @var FormValidation $oFormValidation */ $oFormValidation = Factory::service('FormValidation'); - for ($i = 0; $i < $this->data['num_questions']; $i++) { - - $oFormValidation->set_rules( - 'question[' . $i . '][question]', - '', - 'required|is_natural_no_zero' - ); + $aRules = []; - $oFormValidation->set_rules( - 'question[' . $i . '][answer]', - '', - 'trim|required' - ); + for ($i = 0; $i < $this->data['num_questions']; $i++) { + $aRules['question[' . $i . '][question]'] = [ + FormValidation::RULE_REQUIRED, + FormValidation::RULE_IS_NATURAL_NO_ZERO, + ]; + $aRules['question[' . $i . '][answer]'] = ['trim', FormValidation::RULE_REQUIRED]; } for ($i = 0; $i < $this->data['num_custom_questions']; $i++) { + $aRules['custom_question[' . $i . '][question]'] = ['trim', FormValidation::RULE_REQUIRED]; + $aRules['custom_question[' . $i . '][answer]'] = ['trim', FormValidation::RULE_REQUIRED]; + } - $oFormValidation->set_rules( - 'custom_question[' . $i . '][question]', - '', - 'trim|required' - ); + try { - $oFormValidation->set_rules( - 'custom_question[' . $i . '][answer]', - '', - 'trim|required' + $oValidator = $oFormValidation->buildValidator( + $aRules, + [FormValidation::RULE_IS_NATURAL_NO_ZERO => lang('fv_required')] ); - } - - $oFormValidation->set_message('required', lang('fv_required')); - $oFormValidation->set_message('is_natural_no_zero', lang('fv_required')); + $oValidator->run(); - if ($oFormValidation->run()) { + // The validated data carries the trimmed values + $aPost = $oValidator->getValidatedData(); // Make sure that we have different questions $aQuestionIndex = []; - $aQuestion = array_filter((array) $oInput->post('question', true)); + $aQuestion = array_filter((array) ($aPost['question'] ?? [])); $bError = false; foreach ($aQuestion as $q) { @@ -163,7 +155,7 @@ public function index() } $aQuestionIndex = []; - $aQuestion = array_filter((array) $oInput->post('custom_question', true)); + $aQuestion = array_filter((array) ($aPost['custom_question'] ?? [])); foreach ($aQuestion as $q) { if (array_search($q['question'], $aQuestionIndex) === false) { @@ -179,9 +171,9 @@ public function index() // Good arrows. Save questions $aData = []; - if ($oInput->post('question', true)) { + if (!empty($aPost['question'])) { - foreach ($oInput->post('question', true) as $q) { + foreach ($aPost['question'] as $q) { $oTemp = new stdClass(); @@ -196,8 +188,8 @@ public function index() } } - if ($oInput->post('custom_question', true)) { - foreach ((array) $oInput->post('custom_question', true) as $aQuestion) { + if (!empty($aPost['custom_question'])) { + foreach ((array) $aPost['custom_question'] as $aQuestion) { $aData[] = (object) [ 'question' => trim($aQuestion['question']), 'answer' => $aQuestion['answer'], @@ -228,8 +220,8 @@ public function index() $this->oUserFeedback->error(lang('auth_twofactor_question_unique')); } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); } } diff --git a/auth/controllers/PasswordForgotten.php b/auth/controllers/PasswordForgotten.php index 58eed802..e41c79a2 100644 --- a/auth/controllers/PasswordForgotten.php +++ b/auth/controllers/PasswordForgotten.php @@ -17,12 +17,12 @@ use Nails\Auth\Model\User; use Nails\Auth\Model\User\Password; use Nails\Auth\Service\Authentication; +use Nails\Auth\Validator\User\Identifier; use Nails\Common\Exception\Encrypt\DecodeException; use Nails\Common\Exception\EnvironmentException; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\NailsException; use Nails\Common\Service\Config; -use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; use Nails\Common\Service\Uri; use Nails\Factory; @@ -53,8 +53,6 @@ public function index() { /** @var Input $oInput */ $oInput = Factory::service('Input'); - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); /** @var Config $oConfig */ $oConfig = Factory::service('Config'); /** @var Password $oUserPasswordModel */ @@ -86,17 +84,7 @@ public function index() // -------------------------------------------------------------------------- - $aRules = array_filter([ - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'EMAIL' ? ['required', 'valid_email'] : null, - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'USERNAME' ? ['required'] : null, - \Nails\Config::get('APP_NATIVE_LOGIN_USING') === 'BOTH' ? ['required'] : null, - ]); - - $oFormValidation - ->buildValidator([ - 'identifier' => reset($aRules), - ]) - ->run(); + (new Identifier())->run($oInput->post()); if (appSetting('user_password_reset_captcha_enabled', 'auth')) { if (!$oCaptchaService->verify()) { diff --git a/auth/controllers/Register.php b/auth/controllers/Register.php index d5432b8e..8054a061 100644 --- a/auth/controllers/Register.php +++ b/auth/controllers/Register.php @@ -15,6 +15,7 @@ use Nails\Auth\Model\User\Group; use Nails\Auth\Model\User\Password; use Nails\Auth\Service\SocialSignOn; +use Nails\Auth\Validator\User\Identity; use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; use Nails\Common\Service\Session; @@ -57,8 +58,6 @@ public function index() { /** @var Session $oSession */ $oSession = Factory::service('Session'); - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); /** @var Input $oInput */ $oInput = Factory::service('Input'); /** @var \Nails\Auth\Model\User $oUserModel */ @@ -88,42 +87,13 @@ public function index() try { - $oFormValidation - ->buildValidator([ - 'first_name' => [$oFormValidation::RULE_REQUIRED], - 'last_name' => [$oFormValidation::RULE_REQUIRED], - 'password' => [$oFormValidation::RULE_REQUIRED], - 'email' => in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['EMAIL', 'BOTH']) ? [ - $oFormValidation::RULE_REQUIRED, - $oFormValidation::RULE_VALID_EMAIL, - $oFormValidation::rule( - $oFormValidation::RULE_IS_UNIQUE, $oUserEmailModel->getTableName(), 'email' - ), - ] : [], - 'username' => in_array(Config::get('APP_NATIVE_LOGIN_USING'), [ - 'USERNAME', - 'BOTH', - ]) ? [ - $oFormValidation::RULE_REQUIRED, - $oFormValidation::rule( - $oFormValidation::RULE_IS_UNIQUE, $oUserModel->getTableName(), 'username' - ), - ] : [], + (new Identity()) + ->addRules([ + 'first_name' => [FormValidation::RULE_REQUIRED], + 'last_name' => [FormValidation::RULE_REQUIRED], + 'password' => [FormValidation::RULE_REQUIRED], ]) - ->setMessages([ - $oFormValidation::RULE_IS_UNIQUE => implode('', [ - Config::get('APP_NATIVE_LOGIN_USING') === 'EMAIL' - ? lang('auth_register_email_is_unique', siteUrl('auth/password/forgotten')) - : null, - Config::get('APP_NATIVE_LOGIN_USING') === 'USERNAME' - ? lang('auth_register_username_is_unique', siteUrl('auth/password/forgotten')) - : null, - Config::get('APP_NATIVE_LOGIN_USING') === 'BOTH' - ? lang('auth_register_identity_is_unique', siteUrl('auth/password/forgotten')) - : null, - ]), - ]) - ->run(); + ->run($oInput->post()); if (appSetting('user_registration_captcha_enabled', 'auth')) { if (!$oCaptchaService->verify()) { diff --git a/auth/views/email/user_import_complete.php b/auth/views/email/user_import_complete.php new file mode 100644 index 00000000..d4652ea3 --- /dev/null +++ b/auth/views/email/user_import_complete.php @@ -0,0 +1,49 @@ +

    + The user import you requested has finished. +

    + + + + + + + + + + + + + + + + + + + +
    Rows in file{{import.row_count}}
    Accounts created{{import.success}}
    Rows with warnings{{import.warnings}}
    Rows with errors{{import.errors}}
    +{{#error}} +
    {{error}}
    +{{/error}} +{{#import.errors}} +

    + Some rows could not be imported. The log records what happened to every row + in the file, so you can correct the failures and import them again. +

    +{{/import.errors}} +{{#import.warnings}} +

    + Some accounts were created, but something which had to happen afterwards did + not. They exist and can be signed in to; the log identifies them, and + whatever was meant to follow needs finishing by hand. +

    +{{/import.warnings}} +{{#log_url}} +

    + Download the log +

    +{{/log_url}} +{{#import.url}} +

    + View this import in admin +

    +{{/import.url}} diff --git a/auth/views/email/user_import_complete_plaintext.php b/auth/views/email/user_import_complete_plaintext.php new file mode 100644 index 00000000..77f68de3 --- /dev/null +++ b/auth/views/email/user_import_complete_plaintext.php @@ -0,0 +1,29 @@ +The user import you requested has finished. + +Rows in file: {{import.row_count}} +Accounts created: {{import.success}} +Rows with warnings: {{import.warnings}} +Rows with errors: {{import.errors}} +{{#error}} +{{error}} +{{/error}} + +{{#import.errors}} +Some rows could not be imported. The log records what happened to every row in the file, so you can correct the failures and import them again. +{{/import.errors}} + +{{#import.warnings}} +Some accounts were created, but something which had to happen afterwards did not. They exist and can be signed in to; the log identifies them, and whatever was meant to follow needs finishing by hand. +{{/import.warnings}} + +{{#log_url}} +Download the log (you will need to be signed in to admin): + +{{log_url}} +{{/log_url}} +{{#import.url}} + +View this import in admin: + +{{import.url}} +{{/import.url}} diff --git a/auth/views/email/user_import_failed.php b/auth/views/email/user_import_failed.php new file mode 100644 index 00000000..f6da73d9 --- /dev/null +++ b/auth/views/email/user_import_failed.php @@ -0,0 +1,29 @@ +

    + The user import you requested was rejected; no user accounts were created. +

    +{{#error}} +
    {{error}}
    +{{/error}} +{{#import.errors}} +

    + The log records every row which could not be imported, and why. Correct them + in your CSV and upload it again. +

    +{{/import.errors}} +{{#import.warnings}} +

    + Some accounts were created before the import stopped, but something which had + to happen afterwards did not. They exist; the log identifies them, and + whatever was meant to follow needs finishing by hand. +

    +{{/import.warnings}} +{{#log_url}} +

    + Download the log +

    +{{/log_url}} +{{#import.url}} +

    + View this import in admin +

    +{{/import.url}} diff --git a/auth/views/email/user_import_failed_plaintext.php b/auth/views/email/user_import_failed_plaintext.php new file mode 100644 index 00000000..eebe63fe --- /dev/null +++ b/auth/views/email/user_import_failed_plaintext.php @@ -0,0 +1,23 @@ +The user import you requested was rejected; no user accounts were created. + +{{error}} +{{#import.errors}} + +The log records every row which could not be imported, and why. Correct them in your CSV and upload it again. +{{/import.errors}} +{{#import.warnings}} + +Some accounts were created before the import stopped, but something which had to happen afterwards did not. They exist; the log identifies them, and whatever was meant to follow needs finishing by hand. +{{/import.warnings}} +{{#log_url}} + +Download the log (you will need to be signed in to admin): + +{{log_url}} +{{/log_url}} +{{#import.url}} + +View this import in admin: + +{{import.url}} +{{/import.url}} diff --git a/composer.json b/composer.json index 1ed8f7f2..0f19ac59 100644 --- a/composer.json +++ b/composer.json @@ -32,8 +32,10 @@ "php": ">=8.3", "nails/common": "dev-develop", "nails/module-admin": "dev-develop", + "nails/module-api": "dev-develop", "nails/module-captcha": "dev-develop", "nails/module-console": "dev-develop", + "nails/module-cron": "dev-develop", "nails/module-email": "dev-develop", "nails/module-form-builder": "dev-develop", "hybridauth/hybridauth": "~3.0", @@ -44,7 +46,11 @@ }, "require-dev": { "phpunit/phpunit": "^12.0", - "phpstan/phpstan": "^2.0" + "phpstan/phpstan": "^2.0", + "nails/module-queue": "dev-develop" + }, + "suggest": { + "nails/module-queue": "Processes user imports on a long-running worker, rather than in chunks on the cron." }, "scripts": { "test": "./vendor/bin/phpunit", @@ -55,6 +61,11 @@ "Nails\\Auth\\": "src/" } }, + "autoload-dev": { + "psr-4": { + "Tests\\Auth\\": "tests/" + } + }, "extra": { "nails": { "moduleName": "auth", diff --git a/phpunit.xml b/phpunit.xml index fb22583f..d3c25537 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -4,4 +4,7 @@ ./tests + + + diff --git a/services/services.php b/services/services.php index 99ef2c95..e313bb46 100644 --- a/services/services.php +++ b/services/services.php @@ -42,6 +42,34 @@ return new Service\User\Import(); } }, + 'UserImportCsv' => function (): Service\User\Import\Csv { + if (class_exists('\App\Auth\Service\User\Import\Csv')) { + return new \App\Auth\Service\User\Import\Csv(); + } else { + return new Service\User\Import\Csv(); + } + }, + 'UserImportDispatcher' => function (): Service\User\Import\Dispatcher { + if (class_exists('\App\Auth\Service\User\Import\Dispatcher')) { + return new \App\Auth\Service\User\Import\Dispatcher(); + } else { + return new Service\User\Import\Dispatcher(); + } + }, + 'UserImportProcessor' => function (): Service\User\Import\Processor { + if (class_exists('\App\Auth\Service\User\Import\Processor')) { + return new \App\Auth\Service\User\Import\Processor(); + } else { + return new Service\User\Import\Processor(); + } + }, + 'UserImportValidator' => function (): Service\User\Import\Validator { + if (class_exists('\App\Auth\Service\User\Import\Validator')) { + return new \App\Auth\Service\User\Import\Validator(); + } else { + return new Service\User\Import\Validator(); + } + }, 'UserMeta' => function (): Service\User\Meta { if (class_exists('\App\Auth\Service\User\Meta')) { return new \App\Auth\Service\User\Meta(); @@ -93,6 +121,20 @@ return new Model\User\Group(); } }, + 'UserImport' => function (): Model\User\Import { + if (class_exists('\App\Auth\Model\User\Import')) { + return new \App\Auth\Model\User\Import(); + } else { + return new Model\User\Import(); + } + }, + 'UserImportItem' => function (): Model\User\Import\Item { + if (class_exists('\App\Auth\Model\User\Import\Item')) { + return new \App\Auth\Model\User\Import\Item(); + } else { + return new Model\User\Import\Item(); + } + }, 'UserPassword' => function (): Model\User\Password { // @todo (Pablo 2025-07-15) - this should be a service if (class_exists('\App\Auth\Model\User\Password')) { @@ -145,6 +187,20 @@ return new Factory\Email\PasswordUpdated(); } }, + 'EmailUserImportComplete' => function (): Factory\Email\User\Import\Complete { + if (class_exists('\App\Auth\Factory\Email\User\Import\Complete')) { + return new \App\Auth\Factory\Email\User\Import\Complete(); + } else { + return new Factory\Email\User\Import\Complete(); + } + }, + 'EmailUserImportFailed' => function (): Factory\Email\User\Import\Failed { + if (class_exists('\App\Auth\Factory\Email\User\Import\Failed')) { + return new \App\Auth\Factory\Email\User\Import\Failed(); + } else { + return new Factory\Email\User\Import\Failed(); + } + }, 'EmailVerifyEmail' => function (): Factory\Email\VerifyEmail { if (class_exists('\App\Auth\Factory\Email\VerifyEmail')) { return new \App\Auth\Factory\Email\VerifyEmail(); @@ -204,6 +260,20 @@ return new Resource\User\Group($resource, $model); } }, + 'UserImport' => function ($resource, $model): Resource\User\Import { + if (class_exists('\App\Auth\Resource\User\Import')) { + return new \App\Auth\Resource\User\Import($resource, $model); + } else { + return new Resource\User\Import($resource, $model); + } + }, + 'UserImportItem' => function ($resource, $model): Resource\User\Import\Item { + if (class_exists('\App\Auth\Resource\User\Import\Item')) { + return new \App\Auth\Resource\User\Import\Item($resource, $model); + } else { + return new Resource\User\Import\Item($resource, $model); + } + }, 'UserPasswordHistory' => function ($resource, $model): Resource\User\Password\History { if (class_exists('\App\Auth\Resource\User\Password\History')) { return new \App\Auth\Resource\User\Password\History($resource, $model); diff --git a/src/Admin/Controller/Accounts.php b/src/Admin/Controller/Accounts.php index bbca9ce8..175b485b 100755 --- a/src/Admin/Controller/Accounts.php +++ b/src/Admin/Controller/Accounts.php @@ -22,6 +22,7 @@ use Nails\Auth\Model\User; use Nails\Auth\Model\User\Group; use Nails\Auth\Model\User\Password; +use Nails\Auth\Validator\User\Identity; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\ModelException; use Nails\Common\Exception\NailsException; @@ -274,7 +275,7 @@ public function __construct() // -------------------------------------------------------------------------- - get_instance()->lang->load('admin_accounts'); + Factory::service('Translation')->load('admin_accounts'); /** @var ChangeLog oChangeLogModel */ $this->oChangeLogModel = Factory::model('ChangeLog', \Nails\Admin\Constants::MODULE_SLUG); } @@ -363,32 +364,19 @@ public function create(): void if ($oInput->post()) { - /** @var FormValidation $oFormValidation */ - $oFormValidation = Factory::service('FormValidation'); - - // Set rules - $oFormValidation->set_rules('group_id', '', 'required|is_natural_no_zero'); - $oFormValidation->set_rules('password', '', ''); - $oFormValidation->set_rules('send_activation', '', ''); - $oFormValidation->set_rules('temp_pw', '', ''); - $oFormValidation->set_rules('first_name', '', 'required|max_length[150]'); - $oFormValidation->set_rules('last_name', '', 'required|max_length[150]'); - $oFormValidation->set_rules('email', '', 'required|valid_email|is_unique[' . Config::get('NAILS_DB_PREFIX') . 'user_email.email]|max_length[255]'); - - if (in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['BOTH', 'USERNAME'])) { - $oFormValidation->set_rules('username', '', 'required|max_length[150]|alpha_dash_period|is_unique[' . Config::get('NAILS_DB_PREFIX') . 'user.username]'); - } + try { - // Set messages - $oFormValidation->set_message('required', lang('fv_required')); - $oFormValidation->set_message('min_length', lang('fv_min_length')); - $oFormValidation->set_message('alpha_dash_period', lang('fv_alpha_dash_period')); - $oFormValidation->set_message('is_natural_no_zero', lang('fv_required')); - $oFormValidation->set_message('valid_email', lang('fv_valid_email')); - $oFormValidation->set_message('is_unique', lang('fv_email_already_registered')); + (new Identity()) + ->addRules([ + 'group_id' => [FormValidation::RULE_REQUIRED, FormValidation::RULE_IS_NATURAL_NO_ZERO], + 'first_name' => [FormValidation::RULE_REQUIRED, FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150)], + 'last_name' => [FormValidation::RULE_REQUIRED, FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150)], + ]) + ->setMessages([ + FormValidation::RULE_IS_NATURAL_NO_ZERO => lang('fv_required'), + ]) + ->run($oInput->post()); - // Execute - if ($oFormValidation->run()) { // Success $aData = [ @@ -463,8 +451,8 @@ public function create(): void )); } - } else { - $this->oUserFeedback->error(lang('fv_there_were_errors')); + } catch (ValidationException $e) { + $this->oUserFeedback->error($e->getMessage()); } } diff --git a/src/Admin/Controller/Import.php b/src/Admin/Controller/Import.php index 002eb085..377f5101 100644 --- a/src/Admin/Controller/Import.php +++ b/src/Admin/Controller/Import.php @@ -1,7 +1,15 @@ 'import-user', - 'is_hidden' => true, - 'allowed_types' => 'csv', - ]; + /** + * The bucket the source CSV and its log are stored in + * + * @var array + */ + const array IMPORT_BUCKET = Processor::IMPORT_BUCKET; + + /** + * The permission required to work with imports + * + * @var string + */ + const PERMISSION = Permission\Users\Create::class; + + /** + * Where the controller lives + * + * @var string + */ + const URL = 'admin/auth/import'; + + /** + * The number of per-line errors reported when an upload is rejected + * + * A wholly malformed file produces one per row; the first few say everything + * the thousandth does. Matches Processor::ERROR_SAMPLE_SIZE. + * + * @var int + */ + const ERROR_SAMPLE_SIZE = Processor::ERROR_SAMPLE_SIZE; + + /** + * How long the expiring CDN URL a download hands out is valid for, in seconds + * + * Shorter than module-admin's ADMIN_DATA_EXPORT_URL_TTL (300, see + * Nails\Admin\Service\DataExport::EXPORT_TTL) because the token only has to + * survive the 302 hop; a visitor who was not signed in re-enters log() after + * logging in and is issued a fresh one. + * + * @var int + */ + const int URL_TTL = 60; // -------------------------------------------------------------------------- @@ -62,63 +114,69 @@ public static function announce(): Nav|array|null * * @return void * @throws FactoryException + * @throws ModelException */ public function index(): void { - if (!userHasPermission(Permission\Users\Create::class)) { - unauthorised(); - } - - // -------------------------------------------------------------------------- + $this->assertPermission(); + $this->assertRunning(); /** @var Input $oInput */ $oInput = Factory::service('Input'); /** @var \Nails\Auth\Service\User\Import $oImportService */ $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - if ($oInput->post()) { - try { + try { + /** + * On GET as well as POST: a template which cannot identify an account + * must not be offered for upload in the first place. + */ + $this->assertTemplate(); - if ($oInput->post('action') === 'preview') { - $this - ->validateUpload() - ->renderPreview( - $this->uploadCsv() - ); - - return; + } catch (TemplateException $e) { + $this->oUserFeedback->error($this->escape($e->getMessage())); + $this->data['bTemplateUnusable'] = true; + } - } elseif ($oInput->post('action') === 'import') { - $this - ->validateObject() - ->processImport(); + if ($oInput->post() && empty($this->data['bTemplateUnusable'])) { + try { - } else { - throw new \Exception('Unrecognised action'); - } + $this->handleUpload(); + return; } catch (ValidationException $e) { - $this->oUserFeedback->error( - sprintf( - '%s:
    %s
    ', - $e->getMessage(), + + /** + * Escaped on the way in: UserFeedback messages are rendered raw + * so that the markup below survives, and both the message and + * the itemised errors quote cell and header values lifted + * straight out of the uploaded CSV. + */ + $sMessage = $this->escape($e->getMessage()); + $aErrors = $e->getData() ?? []; + + if (!empty($aErrors)) { + $sMessage .= sprintf( + ':
    %s
    ', implode(';', [ 'max-height: 10rem', 'overflow: auto', 'margin-bottom: 0;', ]), - implode('
    ', $e->getData() ?? []) - ) - ); + implode('
    ', array_map([$this, 'escape'], $aErrors)) + ); + } + + $this->oUserFeedback->error($sMessage); } catch (\Exception $e) { - $this->oUserFeedback->error($e->getMessage()); + $this->oUserFeedback->error($this->escape($e->getMessage())); } } // -------------------------------------------------------------------------- - $this->data['page']->title = 'Import Users'; + $this->data['page']->title = 'Users › Import'; $this->data['additionalFields'] = $oImportService->getAdditionalFields(); Helper::loadView('index'); } @@ -133,6 +191,9 @@ public function index(): void */ public function template(): void { + $this->assertPermission(); + $this->assertTemplate(); + /** @var \Nails\Auth\Service\User\Import $oImportService */ $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); @@ -165,269 +226,349 @@ public function template(): void // -------------------------------------------------------------------------- /** - * Validates the CSV file upload + * Renders the shell of the preview; the rows themselves are paged in from + * the API * - * @return $this + * @return void * @throws FactoryException * @throws ModelException - * @throws NailsException - * @throws ValidationException */ - protected function validateUpload(): self + public function preview(): void { - /** @var Input $oInput */ - $oInput = Factory::service('Input'); - /** @var Cdn\Service\Cdn $oCdn */ - $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + $this->assertPermission(); + $this->assertRunning(); - $aFile = $oInput::file('csv'); - if (empty($aFile)) { - throw new ValidationException('No file selected for upload'); - } + $oImport = $this->getImport(); - if ($aFile['error'] !== UPLOAD_ERR_OK) { - throw new ValidationException( - $oCdn::getUploadError($aFile['error']) - ); - } + /** @var \Nails\Auth\Service\User\Import $oImportService */ + $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - $sMime = $oCdn->getMimeFromFile($aFile['tmp_name']); - if ($aFile['type'] !== 'text/csv' && $sMime !== 'text/plain') { - throw new ValidationException( - 'Uploaded file is not a CSV' - ); + try { + $aKeys = $this->getHeader($oImport); + } catch (ValidationException $e) { + // The CSV has gone; show the columns we would have expected + $this->oUserFeedback->error($this->escape($e->getMessage())); + $aKeys = []; } - $this->validateData( - $this->parseCsv($aFile['tmp_name']) + $this->data['oImport'] = $oImport; + $this->data['bCsvMissing'] = empty($aKeys); + $this->data['aRegistered'] = $this->getRegistered($oImport); + $this->data['aKeys'] = $aKeys ?: $oImportService->getKeys(); + $this->data['sApproveUrl'] = static::URL . '/approve/' . $oImport->id; + $this->data['sListUrl'] = static::URL; + $this->data['page']->title = sprintf( + 'Users › Import › Preview (#%s — %s rows)', + $oImport->id, + $oImport->row_count ?? 0 ); - return $this; + Helper::loadView('preview'); } // -------------------------------------------------------------------------- /** - * @return $this + * Approves a draft import and hands it to a runner + * + * @return void * @throws FactoryException * @throws ModelException - * @throws NailsException - * @throws ValidationException */ - protected function validateObject(): self + public function approve(): void { + $this->assertPermission(); + $this->assertPost(); + /** @var Input $oInput */ $oInput = Factory::service('Input'); - /** @var Cdn\Service\Cdn $oCdn */ - $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); - /** @var Cdn\Model\CdnObject $oObjectModel */ - $oObjectModel = Factory::model('Object', Cdn\Constants::MODULE_SLUG); + /** @var Dispatcher $oDispatcher */ + $oDispatcher = Factory::service('UserImportDispatcher', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); - /** @var Cdn\Resource\CdnObject $oObject */ - $oObject = $oObjectModel->getById((int) $oInput->post('object_id')); - if (empty($oObject)) { - throw new RuntimeException( - 'CDN Object does not exist' - ); - } elseif ($oObject->file->mime !== 'text/csv') { - throw new RuntimeException( - 'Object is not a CSV' - ); - } + $oImport = $this->getImport(); - $sPath = $oCdn->objectLocalPath($oObject->id); - if (empty($sPath)) { - throw new RuntimeException( - 'Failed to get a local path for CSV file.' - ); - } + try { - $this->validateData( - $this->parseCsv($sPath) - ); + if ($oImport->status !== Status::DRAFT) { + throw new ValidationException( + 'Only draft imports can be approved' + ); + } + + $this->assertTemplate(); + + // The file has been sat in the CDN since it was uploaded; make sure + // it is still there, and still makes sense, before committing to it + $oValidator->validateHeader($this->getHeader($oImport)); + + /** + * Registration is state, so it is re-checked here rather than trusted + * from the upload. Without the admin's consent to skip them, a job + * with registered rows would reach a runner only to be rejected, so + * it is refused now while there is somebody to tell. + */ + $bSkipRegistered = (bool) $oInput->post('skip_registered'); + $aRegistered = $this->getRegistered($oImport); + + if (!empty($aRegistered) && !$bSkipRegistered) { + $this->assertNoErrors($this->flattenRowErrors($aRegistered)); + } + + $oModel->update($oImport->id, [ + 'skip_registered' => $bSkipRegistered, + ]); + + $oDispatcher->dispatch($oImport); - return $this; + $this->oUserFeedback->success(sprintf( + 'Import #%s has been queued and will begin shortly.', + $oImport->id + )); + + } catch (\Exception $e) { + $this->oUserFeedback->error($this->escape($e->getMessage())); + } + + redirect(static::URL); } // -------------------------------------------------------------------------- /** - * Validates the CSV data + * Redirects to a short-lived URL for the job's source CSV * - * @param array $aData The data to validate - * - * @return $this + * @return void * @throws FactoryException - * @throws ValidationException * @throws ModelException - * @throws NailsException */ - protected function validateData(array $aData): self + public function source(): void { - /** @var \Nails\Auth\Service\User\Import $oImportService */ - $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - /** @var FormValidation $oFormValidationService */ - $oFormValidationService = Factory::service('FormValidation'); + $this->assertPermission(); + $this->download($this->getImport()->object_id); + } - $aHeader = array_splice($aData, 0, 1); - $aHeader = reset($aHeader); + // -------------------------------------------------------------------------- - // Validate Header Row - if (empty($aHeader)) { - throw new ValidationException( - 'Missing header row' - ); - } + /** + * Redirects to a short-lived URL for the job's log + * + * @return void + * @throws FactoryException + * @throws ModelException + */ + public function log(): void + { + $this->assertPermission(); + $this->download($this->getImport()->log_id); + } - $aKeys = $oImportService->getKeys(); - $aDiff = array_diff($aHeader, $aKeys); - if (!empty($aDiff)) { - throw new ValidationException(sprintf( - 'Header row contains the following invalid values: %s', - implode(', ', $aDiff) - )); - } + // -------------------------------------------------------------------------- - // Validate data - $aErrors = []; - $iLines = 2; // Skip the header + /** + * Redirects to a short-lived URL for one of the job's CDN objects + * + * The source CSV and the log both contain personal data, so neither is ever + * linked directly: the URL is minted here, behind the permission check, and + * expires. It does end up in the address bar and the Referer of the CDN + * request - the same trade module-admin's data export makes - which is why + * the TTL is measured in seconds. + * + * Reading personal data sits behind a create permission because that is the + * permission which governs the whole of this controller; the ability to run + * an import is the ability to read its file. + * + * @param int|null $iObjectId The CDN object to hand out + * + * @return void + */ + protected function download(?int $iObjectId): void + { + if (empty($iObjectId)) { + // No log was attached, or the object has since been destroyed + show404(); + } - // Key the rules by the header's own columns; the CSV need not use every - // key, nor list them in the same order as getKeys() - $aValidationRules = array_filter( - array_map( - fn($sKey) => $oImportService->getValidationRules($sKey), - array_combine($aHeader, $aHeader) - ) - ); + redirect(cdnExpiringUrl($iObjectId, static::URL_TTL, true)); + } - foreach ($aData as $aDatum) { + // -------------------------------------------------------------------------- - try { + /** + * Validates the upload, stores it, and records the job + * + * Nothing is persisted unless the file is usable; a rejected upload leaves + * neither a CDN object nor a job behind. + * + * @throws FactoryException + * @throws ModelException + * @throws NailsException + * @throws ValidationException + */ + protected function handleUpload(): void + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + + $aFile = $this->validateUpload(); + $iRowCount = $oCsv->countRows($aFile['tmp_name']); + $oObject = $this->uploadCsv(); + + $iId = $oModel->create([ + 'object_id' => $oObject->id, + 'additional' => json_encode((object) ($oInput->post('additional') ?: [])), + 'row_count' => $iRowCount, + 'status' => Status::DRAFT->value, + ]); + + if (empty($iId)) { + throw new RuntimeException(sprintf( + 'Failed to record the import; %s', + $oModel->lastError() + )); + } - $aDatum = array_combine($aHeader, $aDatum); - $aDatum = array_map('trim', $aDatum); + redirect(static::URL . '/preview/' . $iId); + } - // Basic validation - $oFormValidationService - ->buildValidator( - aRules: $aValidationRules, - aData: $aDatum - ) - ->run(); + // -------------------------------------------------------------------------- - } catch (ValidationException $e) { + /** + * Validates the CSV file upload + * + * Only whole-of-file concerns are checked here — that there is a file, that + * it is a CSV, that its header is one we understand, and that it does not + * contradict itself. Row level validation is the runner's job. + * + * @return array The $_FILES entry + * @throws FactoryException + * @throws ModelException + * @throws NailsException + * @throws ValidationException + */ + protected function validateUpload(): array + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); + /** @var Cdn\Service\Cdn $oCdn */ + $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); - foreach ($e->getData() as $key => $error) { - $aErrors[] = sprintf( - 'Line %d: %s: %s', - $iLines, - $key, - $error - ); - } + $aFile = $oInput::file('csv'); + if (empty($aFile)) { + throw new ValidationException('No file selected for upload'); + } - } catch (Throwable $e) { - $aErrors[] = sprintf( - 'Error on line %d: %s', - $iLines, - $e->getMessage() - ); - } + if ($aFile['error'] !== UPLOAD_ERR_OK) { + throw new ValidationException( + $oCdn::getUploadError($aFile['error']) + ); + } - $iLines++; + $sMime = $oCdn->getMimeFromFile($aFile['tmp_name']); + if ($aFile['type'] !== 'text/csv' && $sMime !== 'text/plain') { + throw new ValidationException( + 'Uploaded file is not a CSV' + ); } - // Duplicate detection cannot live in the per-field rules; those only ever - // see a single value, and the validator abandons a field's remaining rules - // as soon as one of them fails. It's a whole-of-file concern, so handle it - // as one pass over the parsed data. - $aErrors = array_merge( - $aErrors, - $this->detectDuplicates($aHeader, $aData) - ); + $aHeader = $oCsv->getHeader($aFile['tmp_name']); - if (!empty($aErrors)) { + $oValidator->validateHeader($aHeader); - $message = count($aErrors) === 1 - ? '1 error was found in the CSV file' - : sprintf('%d errors were found in the CSV file', count($aErrors)); + $this->assertNoErrors( + $oValidator->detectDuplicates($aFile['tmp_name']) + ); - throw (new ValidationException($message)) - ->setData($aErrors); - } + /** + * Row validation used to be left entirely to the runner, which meant an + * admin could approve - and commit to - a file which could never import. + * The rows are streamed, so this holds no more than one at a time, and + * the request already reads the whole file twice before reaching here. + * + * Only "the CSV is wrong" problems are checked. Values which are already + * registered are a separate, skippable concern handled at preview; see + * Import\Validator::detectRegistered(). + */ + $this->assertNoErrors( + $this->flattenRowErrors( + $oValidator->validateRows($aHeader, $oCsv->readRawRows($aFile['tmp_name'])) + ) + ); - return $this; + return $aFile; } // -------------------------------------------------------------------------- /** - * Detects values which are duplicated within the CSV itself + * Renders per-line errors as flat, prefixed strings * - * @param array $aHeader The CSV's header row - * @param array $aData The CSV's data rows, header removed + * Matches detectDuplicates()'s output so both can go through the same + * reporting path. + * + * @param array $aErrors Errors keyed by line number * * @return string[] - * @throws FactoryException */ - protected function detectDuplicates(array $aHeader, array $aData): array + protected function flattenRowErrors(array $aErrors): array { - /** @var \Nails\Auth\Service\User\Import $oImportService */ - $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - - $aErrors = []; + $aOut = []; - foreach ($oImportService->getUniqueKeys() as $sKey) { - - $iColumn = array_search($sKey, $aHeader, true); - if ($iColumn === false) { - continue; - } - - $aSeen = []; - - foreach ($aData as $iIndex => $aDatum) { - - $sValue = strtolower(trim($aDatum[$iColumn] ?? '')); - if ($sValue === '') { - continue; - } - - $iLine = $iIndex + 2; // Lines are 1 indexed, and the header is line 1 - - if (array_key_exists($sValue, $aSeen)) { - $aErrors[] = sprintf( - 'Line %d: %s: "%s" must only appear once; it is also on line %d', - $iLine, - $sKey, - $sValue, - $aSeen[$sValue] - ); - } else { - $aSeen[$sValue] = $iLine; - } + foreach ($aErrors as $iLine => $aLineErrors) { + foreach ($aLineErrors as $sError) { + $aOut[] = sprintf('Line %d: %s', $iLine, $sError); } } - return $aErrors; + return $aOut; } // -------------------------------------------------------------------------- /** - * Parses the CSV file into an array + * Rejects the upload if anything was found * - * @param string $sPath The path to the CSV + * The list is capped: a wholly malformed file can produce an error for every + * row, and thousands of near-identical lines tell an admin less than the + * first few do. * - * @return array + * @param string[] $aErrors + * + * @throws ValidationException */ - protected function parseCsv(string $sPath): array + protected function assertNoErrors(array $aErrors): void { - return array_map( - fn($line) => str_getcsv($line, escape: ''), - file($sPath) - ); + if (empty($aErrors)) { + return; + } + + $iTotal = count($aErrors); + + $sMessage = $iTotal === 1 + ? '1 error was found in the CSV file' + : sprintf('%s errors were found in the CSV file', number_format($iTotal)); + + if ($iTotal > static::ERROR_SAMPLE_SIZE) { + $aErrors = array_slice($aErrors, 0, static::ERROR_SAMPLE_SIZE); + $aErrors[] = sprintf( + '…and %s more', + number_format($iTotal - static::ERROR_SAMPLE_SIZE) + ); + } + + throw (new ValidationException($sMessage)) + ->setData($aErrors); } // -------------------------------------------------------------------------- @@ -447,12 +588,12 @@ protected function uploadCsv(): Cdn\Resource\CdnObject $oObject = $oCdn->objectCreate( 'csv', - self::IMPORT_BUCKET, + static::IMPORT_BUCKET, [ 'Content-Type' => 'text/csv', 'metadata' => [ [ - 'key' => (new SystemKey\UserImport)->get(), + 'key' => (new SystemKey\UserImport())->get(), 'value' => true, ], ], @@ -474,209 +615,178 @@ protected function uploadCsv(): Cdn\Resource\CdnObject // -------------------------------------------------------------------------- /** - * @param Cdn\Resource\CdnObject $oObject + * Returns the import named by the URL, or 404s * - * @return void * @throws FactoryException - * @throws NailsException + * @throws ModelException */ - protected function renderPreview(Cdn\Resource\CdnObject $oObject): void + protected function getImport(): Resource\User\Import { - /** @var \Nails\Auth\Service\User\Import $oImportService */ - $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - /** @var Input $oInput */ - $oInput = Factory::service('Input'); - /** @var Cdn\Service\Cdn $oCdn */ - $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); - - $sPath = $oCdn->objectLocalPath($oObject->id); - if (empty($sPath)) { - throw new RuntimeException( - 'Failed to get a local path for CSV file' - ); - } + /** @var Uri $oUri */ + $oUri = Factory::service('Uri'); + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); - $aData = $this->parseCsv($sPath); + /** @var Resource\User\Import|null $oImport */ + $oImport = $oModel->getById((int) $oUri->segment(5)); - $aKeys = $oImportService->getKeys(); - $aHeader = array_splice($aData, 0, 1); - $aHeader = reset($aHeader); - - foreach ($aData as &$aDatum) { - $aDatum = array_combine($aHeader, $aDatum); - foreach ($aKeys as $sField) { - if ($aDatum[$sField] === '') { - $aDatum[$sField] = $oImportService->getDefaultValue($sField); - } - } + if (empty($oImport)) { + show404(); } - $this->data['aKeys'] = $aKeys; - $this->data['aHeader'] = $aHeader; - $this->data['aData'] = $aData; - $this->data['oObject'] = $oObject; - $this->data['oAdditional'] = (object) $oInput->post('additional'); + return $oImport; + } - $this->data['page']->title = 'Import Users: Preview (' . count($aData) . ')'; + // -------------------------------------------------------------------------- - Helper::loadView('preview'); + /** + * Returns the header row of an import's CSV + * + * @return string[] + * @throws FactoryException + * @throws ValidationException + */ + protected function getHeader(Resource\User\Import $oImport): array + { + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + + return $oCsv->getHeader($this->getCsvPath($oImport)); } // -------------------------------------------------------------------------- /** - * Process the import + * Returns the local path of an import's CSV * - * @return void * @throws FactoryException - * @throws NailsException + * @throws ValidationException If the file is no longer retrievable */ - protected function processImport(): void + protected function getCsvPath(Resource\User\Import $oImport): string { - /** @var \Nails\Auth\Service\User\Import $oImportService */ - $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); - /** @var Input $oInput */ - $oInput = Factory::service('Input'); /** @var Cdn\Service\Cdn $oCdn */ $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); - /** @var User $oUserModel */ - $oUserModel = Factory::model('User', Constants::MODULE_SLUG); - $iObjectId = (int) $oInput->post('object_id'); - $oAdditional = json_decode($oInput->post('additional')); - - $sPath = $oCdn->objectLocalPath($iObjectId); + $sPath = $oCdn->objectLocalPath($oImport->object_id); if (empty($sPath)) { - throw new RuntimeException( - 'Failed to get a local path for CSV file.' + throw new ValidationException( + 'Failed to get a local path for the CSV file' ); } - $aKeys = $oImportService->getKeys(); - $aData = $this->parseCsv($sPath); - $aHeader = array_splice($aData, 0, 1); - $aHeader = reset($aHeader); - $iSuccess = 0; - $iError = 0; - $aLog = []; - - foreach ($aData as $aDatum) { - - $aDatum = array_combine($aHeader, $aDatum); - $bSendEmail = stringToBoolean($aDatum['send_email'] ?? false); - - $aUserData = []; - foreach ($aKeys as $sKey) { - $aUserData[$sKey] = $aDatum[$sKey] ?? null; - if ($aUserData[$sKey] === '') { - $aUserData[$sKey] = $oImportService->getDefaultValue($sKey); - } - } + return $sPath; + } - // Apply additional fields - foreach ($oAdditional as $oProperty => $mValue) { - $aUserData[$oProperty] = $oImportService->parseAdditionalFields($oProperty, $mValue); - } + // -------------------------------------------------------------------------- - try { + /** + * @throws FactoryException + */ + /** + * Escapes a value destined for a user feedback message + * + * UserFeedback messages are rendered raw - see module-admin's + * page-header.php - and every message this controller reports quotes header + * or cell values taken from an uploaded CSV, so they are escaped here rather + * than trusted downstream.§ + */ + protected function escape(?string $sValue): string + { + return htmlspecialchars((string) $sValue, ENT_QUOTES, 'UTF-8'); + } - $oUser = $oUserModel->create($aUserData, $bSendEmail); - - if ($oUser) { - $iSuccess++; - $aLog[] = array_merge( - $aDatum, - [ - 'id' => $oUser->id, - 'status' => 'SUCCESS', - 'message' => '', - ] - ); - } else { - $iError++; - $aLog[] = array_merge( - $aDatum, - [ - 'id' => null, - 'status' => 'ERROR', - 'message' => $oUserModel->lastError(), - ] - ); - } + // -------------------------------------------------------------------------- - } catch (Throwable $e) { - $iError++; - $aLog[] = array_merge( - $aDatum, - [ - 'id' => null, - 'status' => 'ERROR', - 'message' => $e->getMessage(), - ] - ); - } + /** + * Returns the job's rows whose unique values are already registered + * + * Recomputed rather than stored: registration is state which can change + * between upload and approval, and a stale answer here would either block a + * job which is now fine or wave through one which is not. + * + * Only meaningful before the job runs, so a job which is past DRAFT is not + * made to pay for the pass. + * + * @return array Errors keyed by line number + * @throws FactoryException + */ + protected function getRegistered(Resource\User\Import $oImport): array + { + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + + if ($oImport->status !== Status::DRAFT) { + return []; } - array_unshift($aLog, array_merge( - $aHeader, - [ - 'id', - 'status', - 'message', - ] - )); + try { + $sPath = $this->getCsvPath($oImport); - // Convert array to CSV and save to cDN - $fp = fopen('php://temp', 'r+'); - foreach ($aLog as $row) { - fputcsv($fp, $row, escape: ''); + } catch (ValidationException $e) { + // The CSV has gone; the caller has already reported that + return []; } - rewind($fp); - $csvLog = stream_get_contents($fp); - fclose($fp); - - /** @var \DateTime $oNow */ - $oNow = Factory::factory('DateTime'); - $oLog = $oCdn->objectCreate( - $csvLog, - self::IMPORT_BUCKET, - [ - 'no-md5-check' => true, - 'Content-Type' => 'text/csv', - 'filename_display' => sprintf( - 'user-import-log-%s.csv', - $oNow->format('Y-m-d_H-i-s') - ), - 'metadata' => [ - [ - 'key' => (new SystemKey\UserImport)->get(), - 'value' => true, - ], - [ - 'key' => (new SystemKey\ImportedFrom)->get(), - 'value' => $iObjectId, - ], - ], - ], - true + + return $oValidator->detectRegistered( + $oCsv->getHeader($sPath), + $oCsv->readRawRows($sPath) ); + } - if (!empty($iSuccess)) { - $this->oUserFeedback->success(sprintf( - '%s user accounts created successfully. See log for details.', - $iSuccess, - cdnServe($oLog->id, true) - )); + // -------------------------------------------------------------------------- + + /** + * Asserts the import template can produce a usable account + * + * @throws TemplateException + * @throws FactoryException + */ + protected function assertTemplate(): void + { + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + protected function assertPermission(): void + { + if (!userHasPermission(static::PERMISSION)) { + unauthorised(); } + } - if (!empty($iError)) { - $this->oUserFeedback->error(sprintf( - '%s user accounts encountered errors. See log for details.', - $iError, - cdnServe($oLog->id, true) - )); + protected function assertRunning(): void + { + /** @var Dispatcher $oDispatcher */ + $oDispatcher = Factory::service('UserImportDispatcher', Constants::MODULE_SLUG); + + if (!$oDispatcher->isRunning()) { + $this->oUserFeedback->warning( + 'The user import cron job is not running' . + '
    The cron job has not been executed within the past 5 minutes; imports will not be processed.' + ); } + } + + // -------------------------------------------------------------------------- + + /** + * State changes are POST only so they cannot be triggered by a link + * + * @throws FactoryException + */ + protected function assertPost(): void + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); - redirect(self::url()); + if (strtoupper((string) $oInput::server('REQUEST_METHOD')) !== 'POST') { + show404(); + } } } diff --git a/src/Admin/Controller/Merge.php b/src/Admin/Controller/Merge.php index dbe07950..f29228b6 100644 --- a/src/Admin/Controller/Merge.php +++ b/src/Admin/Controller/Merge.php @@ -22,6 +22,7 @@ use Nails\Common\Exception\ValidationException; use Nails\Common\Service\FormValidation; use Nails\Common\Service\Input; +use Nails\Common\Validation\Context; use Nails\Factory; use stdClass; @@ -88,12 +89,12 @@ public function index(): void ], 'merge_ids' => [ $oFormValidation::RULE_REQUIRED, - function ($mInput) use ($oInput) { + function ($mInput, Context $oContext) { $aMergeIds = explode(',', $mInput); if (in_array(activeUser('id'), $aMergeIds)) { throw new ValidationException('You cannot list yourself as a user to merge.'); - } elseif (in_array($oInput->post('user_id'), $aMergeIds)) { + } elseif (in_array($oContext->getValue('user_id'), $aMergeIds)) { throw new ValidationException('You cannot merge the target user into itself.'); } }, diff --git a/src/Api/Controller/Import.php b/src/Api/Controller/Import.php new file mode 100644 index 00000000..90c55b96 --- /dev/null +++ b/src/Api/Controller/Import.php @@ -0,0 +1,440 @@ +status->isDeletable() || $oItem->claim_token !== null) + ) { + throw new ApiException( + 'An import which is in progress cannot be deleted', + $oHttpCodes::STATUS_CONFLICT + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Formats the response object + * + * @param Resource\User\Import $oObj The object to format + * + * @throws FactoryException + */ + protected function formatObject($oObj): stdClass + { + return (object) [ + 'id' => $oObj->id, + 'status' => $oObj->status->value, + 'runner' => $oObj->runner?->value, + 'progress' => (object) [ + 'row_count' => $oObj->row_count, + 'validated_count' => $oObj->validated_count, + 'processed_count' => $oObj->processed_count, + 'success_count' => $oObj->success_count, + 'warning_count' => $oObj->warning_count, + 'error_count' => $oObj->error_count, + 'percent' => $oObj->getPercent(), + ], + /** + * Both URLs are the guarded admin routes rather than CDN objects: + * the files carry personal data, so the URL which reaches them is + * minted behind a permission check and expires. + * + * They sit on `source` and `log` rather than on the `urls` object + * below because `item.log.url` is what the admin JS already reads; + * the inconsistency is deliberate. `source.url` has no consumer + * yet, and is exposed so that a future link cannot be tempted back + * to a direct CDN URL. + */ + 'source' => (object) [ + 'id' => $oObj->object?->id, + 'filename' => $oObj->object?->file?->name?->human, + 'url' => siteUrl('admin/auth/import/source/' . $oObj->id), + ], + 'log' => $oObj->log_id + ? (object) [ + 'id' => $oObj->log_id, + 'url' => siteUrl('admin/auth/import/log/' . $oObj->id), + ] + : null, + 'error' => $oObj->error, + /** + * The stored error is deliberately multi-line - see + * Processor::composeError() - so anywhere with room for a single + * line takes the summary rather than doing its own string surgery. + */ + 'error_summary' => $oObj->error + ? explode("\n", $oObj->error, 2)[0] + : null, + 'created' => $oObj->created, + 'started' => $oObj->started, + 'finished' => $oObj->finished, + 'user' => $oObj->user + ? (object) [ + 'id' => $oObj->user->id, + 'name' => $oObj->user->name, + 'email' => $oObj->user->email, + ] + : null, + 'urls' => (object) [ + 'preview' => siteUrl('admin/auth/import/preview/' . $oObj->id), + 'approve' => siteUrl('admin/auth/import/approve/' . $oObj->id), + 'delete' => siteUrl('api/auth/import/' . $oObj->id), + ], + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes a job, and the CDN objects behind it + * + * @param ApiResponse $oApiResponse The API response + * @param Resource\User\Import $oItem The job being deleted + * + * @throws ApiException + * @throws FactoryException + * @throws ModelException + */ + protected function delete(ApiResponse $oApiResponse, Entity $oItem): void + { + // Applies the guard in userCan(), and 500s if the row will not go + parent::delete($oApiResponse, $oItem); + + /** + * Only now that the row is gone: object_id is ON DELETE CASCADE, so + * destroying the CSV first would take the job with it - inside + * objectDestroy()'s own transaction - and the delete above would then + * report a job which no longer exists. + * + * A failure here is logged rather than raised: the row has gone and the + * delete has genuinely succeeded, so there is nothing for the caller to + * retry. It cannot be left silent though - `auth:user:import:clean` + * reaps by walking job rows, so an object orphaned here is unreachable + * by anything else, and the log line is the only thread back to what it + * was for. + */ + /** @var Model\User\Import $oModel */ + $oModel = $this->oModel; + /** @var Logger $oLogger */ + $oLogger = Factory::service('Logger'); + + foreach ($oModel->destroyObjects($oItem) as $iObjectId => $sError) { + $oLogger->error(sprintf( + 'Failed to destroy CDN object #%s belonging to user import #%s; %s', + $iObjectId, + $oItem->id, + $sError + )); + } + } + + // -------------------------------------------------------------------------- + + /** + * Returns a page of the job's CSV, as it will be imported + * + * Reached at GET /api/auth/import/{id}/rows; CrudController::read() routes + * the trailing segment to a same-named method. + * + * @param ApiResponse $oApiResponse The API response + * @param Resource\User\Import $oItem The job being previewed + * + * @throws ApiException + * @throws FactoryException + * @throws NailsException + */ + protected function rows(ApiResponse $oApiResponse, Entity $oItem): void + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); + /** @var HttpCodes $oHttpCodes */ + $oHttpCodes = Factory::service('HttpCodes'); + /** @var Cdn\Service\Cdn $oCdn */ + $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + + $sPath = $oCdn->objectLocalPath($oItem->object->id); + if (empty($sPath)) { + throw new ApiException( + 'The CSV for this import is no longer available', + $oHttpCodes::STATUS_NOT_FOUND + ); + } + + $iPage = max(1, (int) $oInput->get(static::CONFIG_PAGE_PARAM)); + $iOffset = ($iPage - 1) * static::CONFIG_PER_PAGE; + + /** + * row_count is recorded when the file is uploaded, so paging does not + * have to re-scan the file on every request. + */ + $iTotal = $oItem->row_count ?? $oCsv->countRows($sPath); + + $aRows = []; + foreach ($oCsv->readRows($sPath, $iOffset, static::CONFIG_PER_PAGE) as $iLine => $aRow) { + $aRows[] = (object) [ + 'line' => $iLine, + 'data' => (object) $aRow, + ]; + } + + $oApiResponse + ->setData($aRows) + ->setMeta([ + 'header' => $oCsv->getHeader($sPath), + 'pagination' => [ + 'page' => $iPage, + 'per_page' => static::CONFIG_PER_PAGE, + 'total' => $iTotal, + 'previous' => $this->buildUrl($iTotal, $iPage, -1), + 'next' => $this->buildUrl($iTotal, $iPage, 1), + ], + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Returns a page of the job's per-row outcomes + * + * Reached at GET /api/auth/import/{id}/items; CrudController::read() routes + * the trailing segment to a same-named method, as it does for rows(). + * + * Kept off formatObject() deliberately: that runs for every row of the list, + * and the list polls while any job is active, so folding per-row detail into + * it would mean an extra query per job per poll for something nobody is + * looking at until they click. + * + * @param ApiResponse $oApiResponse The API response + * @param Resource\User\Import $oItem The job being inspected + * + * @throws ApiException + * @throws FactoryException + * @throws ModelException + */ + protected function items(ApiResponse $oApiResponse, Entity $oItem): void + { + /** @var Input $oInput */ + $oInput = Factory::service('Input'); + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + $aData = [ + 'where' => [['import_id', $oItem->id]], + 'sort' => [['line', 'ASC']], + ]; + + /** + * A comma-separated list rather than a single value, so the details + * modal can ask for the errors and the warnings - two statuses which + * mean different things to the reader, but which are shown together - in + * one request rather than two. + */ + $aStatuses = $this->readStatuses((string) $oInput->get('status')); + + if (!empty($aStatuses)) { + $aData['where_in'] = [['status', $aStatuses]]; + } + + $iPage = max(1, (int) $oInput->get(static::CONFIG_PAGE_PARAM)); + $iTotal = $oItemModel->countAll($aData); + + $aItems = array_map( + fn(Resource\User\Import\Item $oRow): stdClass => (object) [ + 'line' => $oRow->line, + 'status' => $oRow->status->value, + 'message' => $oRow->message, + 'user_id' => $oRow->user_id, + ], + $oItemModel->getAll($iPage, static::CONFIG_PER_PAGE, $aData) + ); + + $oApiResponse + ->setData($aItems) + ->setMeta([ + 'total_errors' => $oItem->error_count, + 'total_warnings' => $oItem->warning_count, + 'pagination' => [ + 'page' => $iPage, + 'per_page' => static::CONFIG_PER_PAGE, + 'total' => $iTotal, + 'previous' => $this->buildUrl($iTotal, $iPage, -1), + 'next' => $this->buildUrl($iTotal, $iPage, 1), + ], + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Reads the `status` filter + * + * Every value is checked, and an unknown one is refused rather than dropped: + * a filter which quietly ignores what it was asked for would answer a + * question nobody asked. + * + * @param string $sStatus The requested status, or a comma-separated list of them + * + * @return string[] The statuses to filter by, empty for no filter + * @throws ApiException If any of them is not a status + * @throws FactoryException + */ + protected function readStatuses(string $sStatus): array + { + /** @var HttpCodes $oHttpCodes */ + $oHttpCodes = Factory::service('HttpCodes'); + + $aStatuses = []; + + foreach (array_filter(array_map('trim', explode(',', $sStatus))) as $sCandidate) { + + $oStatus = ItemStatus::tryFrom($sCandidate); + + if ($oStatus === null) { + throw new ApiException( + sprintf('"%s" is not a valid item status', $sCandidate), + $oHttpCodes::STATUS_BAD_REQUEST + ); + } + + $aStatuses[] = $oStatus->value; + } + + return array_values(array_unique($aStatuses)); + } +} diff --git a/src/Console/Command/User/Import/Clean.php b/src/Console/Command/User/Import/Clean.php new file mode 100644 index 00000000..4b8d3afc --- /dev/null +++ b/src/Console/Command/User/Import/Clean.php @@ -0,0 +1,261 @@ +setName('auth:user:import:clean') + ->setDescription('Releases orphaned user import jobs and reaps old ones'); + } + + // -------------------------------------------------------------------------- + + /** + * Executes the command + * + * @param InputInterface $oInput The Input Interface provided by Symfony + * @param OutputInterface $oOutput The Output Interface provided by Symfony + */ + protected function execute(InputInterface $oInput, OutputInterface $oOutput): int + { + parent::execute($oInput, $oOutput); + + try { + + $this->banner('User Import: Clean'); + $this + ->releaseOrphans() + ->reapDrafts() + ->rotateFinished(); + + } catch (Throwable $e) { + return $this->abort( + self::EXIT_CODE_FAILURE, + [$e->getMessage()] + ); + } + + $oOutput->writeln(''); + $oOutput->writeln('Complete!'); + + return self::EXIT_CODE_SUCCESS; + } + + // -------------------------------------------------------------------------- + + /** + * Releases claims held by processes which are no longer with us + * + * The status is deliberately left alone; a job resumes from its cursor, and + * sending it back to PENDING would restart it from the top and re-validate + * rows whose users have since been created. + * + * @throws FactoryException + * @throws ModelException + */ + protected function releaseOrphans(): self + { + $this->oOutput->writeln('Releasing orphaned claims'); + + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $oDb + ->set('claim_token', null) + ->set('claimed', null) + ->where('claim_token IS NOT NULL', null, false) + ->where('claimed <', $this->getCutOff('AUTH_USER_IMPORT_STALE_CLAIM', static::STALE_CLAIM)) + ->update($oModel->getTableName()); + + $this->oOutput->writeln(sprintf( + 'Released %s', + $oDb->affected_rows() + )); + + return $this; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes uploads which were never approved + * + * @throws FactoryException + * @throws ModelException + */ + protected function reapDrafts(): self + { + $this->oOutput->writeln(''); + $this->oOutput->writeln('Reaping abandoned drafts'); + + $iDeleted = $this->deleteJobs( + [Status::DRAFT], + $this->getCutOff('AUTH_USER_IMPORT_DRAFT_TTL', static::DRAFT_TTL) + ); + + $this->oOutput->writeln(sprintf('Deleted %s', $iDeleted)); + + return $this; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes jobs which finished long enough ago that nobody is coming back for them + * + * @throws FactoryException + * @throws ModelException + */ + protected function rotateFinished(): self + { + $this->oOutput->writeln(''); + $this->oOutput->writeln('Rotating finished jobs'); + + $iDeleted = $this->deleteJobs( + Status::terminal(), + $this->getCutOff('AUTH_USER_IMPORT_RETENTION', static::RETENTION) + ); + + $this->oOutput->writeln(sprintf('Deleted %s', $iDeleted)); + + return $this; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes jobs of the given statuses which were last modified before the cut off + * + * @param Status[] $aStatuses + * + * @throws FactoryException + * @throws ModelException + */ + protected function deleteJobs(array $aStatuses, string $sCutOff): int + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aRows = $oDb + ->select('id') + ->where_in('status', Status::values($aStatuses)) + ->where('modified <', $sCutOff) + ->order_by('id', 'asc') + ->limit(static::MAX_PER_RUN) + ->get($oModel->getTableName()) + ->result(); + + $iDeleted = 0; + + foreach ($aRows as $oRow) { + + /** @var Resource\User\Import|null $oImport */ + $oImport = $oModel->getById((int) $oRow->id); + if (empty($oImport)) { + continue; + } + + // The job goes first; the CDN objects cascade onto it, and a + // half-deleted job is worse than a lingering file. + if (!$oModel->delete($oImport->id)) { + $this->oOutput->writeln(sprintf( + '↳ Failed to delete import #%s; %s', + $oImport->id, + $oModel->lastError() + )); + continue; + } + + foreach ($oModel->destroyObjects($oImport) as $iObjectId => $sError) { + $this->oOutput->writeln(sprintf( + '↳ Failed to destroy CDN object #%s; %s', + $iObjectId, + $sError + )); + } + + $iDeleted++; + } + + return $iDeleted; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the datetime $sConfigKey seconds ago + * + * @throws FactoryException + */ + protected function getCutOff(string $sConfigKey, int $iDefault): string + { + $iSeconds = (int) Config::get($sConfigKey, $iDefault) ?: $iDefault; + + /** @var \DateTime $oCutOff */ + $oCutOff = Factory::factory('DateTime'); + $oCutOff->sub(new DateInterval('PT' . $iSeconds . 'S')); + + return $oCutOff->format('Y-m-d H:i:s'); + } +} diff --git a/src/Console/Command/User/Import/Process.php b/src/Console/Command/User/Import/Process.php new file mode 100644 index 00000000..2e43d34d --- /dev/null +++ b/src/Console/Command/User/Import/Process.php @@ -0,0 +1,312 @@ +setName('auth:user:import:process') + ->setDescription('Processes any approved user import jobs') + ->addOption( + 'chunk', + 'c', + InputOption::VALUE_REQUIRED, + 'The number of CSV rows to handle at a time' + ) + ->addOption( + 'budget', + 'b', + InputOption::VALUE_REQUIRED, + 'How long, in seconds, to keep processing for' + ); + } + + // -------------------------------------------------------------------------- + + /** + * Executes the command + * + * @param InputInterface $oInput The Input Interface provided by Symfony + * @param OutputInterface $oOutput The Output Interface provided by Symfony + */ + protected function execute(InputInterface $oInput, OutputInterface $oOutput): int + { + parent::execute($oInput, $oOutput); + + try { + + $this->banner('User Import: Process'); + $this->recordLastRun(); + $this->processJobs(); + + } catch (Throwable $e) { + return $this->abort( + self::EXIT_CODE_FAILURE, + [$e->getMessage()] + ); + } + + $oOutput->writeln(''); + $oOutput->writeln('Complete!'); + + return self::EXIT_CODE_SUCCESS; + } + + // -------------------------------------------------------------------------- + + /** + * Records that the cron ran, so admin can warn when it stops + * + * @throws FactoryException + */ + protected function recordLastRun(): void + { + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + setAppSetting( + 'user-import-cron-last-run', + Constants::MODULE_SLUG, + $oNow->format('Y-m-d H:i:s') + ); + } + + // -------------------------------------------------------------------------- + + /** + * Works through the outstanding jobs, oldest first, until the budget is spent + * + * @throws FactoryException + * @throws ModelException + */ + protected function processJobs(): void + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Processor $oProcessor */ + $oProcessor = Factory::service('UserImportProcessor', Constants::MODULE_SLUG); + + $iChunk = (int) ($this->oInput->getOption('chunk') ?: $oProcessor::getChunkSize()); + $iBudget = (int) ($this->oInput->getOption('budget') ?: Config::get('AUTH_USER_IMPORT_RUN_BUDGET', static::RUN_BUDGET)); + $fDeadline = microtime(true) + $iBudget; + + $aSkip = []; + $iFound = 0; + + while (microtime(true) < $fDeadline) { + + $oImport = $this->getNextJob($aSkip); + if (empty($oImport)) { + break; + } + + $iFound++; + $aSkip[] = $oImport->id; + + $sToken = md5(uniqid((string) getmypid(), true)); + + if (!$oModel->claim($oImport->id, $sToken)) { + $this->oOutput->writeln(sprintf( + 'Import #%s is claimed by another process, skipping', + $oImport->id + )); + continue; + } + + $this->oOutput->writeln(sprintf( + 'Processing import #%s (%s)', + $oImport->id, + $oImport->status->value + )); + + try { + + while (!$oImport->status->isTerminal() && microtime(true) < $fDeadline) { + + $sBefore = $this->fingerprint($oImport); + $oImport = $oProcessor->process($oImport, $iChunk); + + $this->oOutput->writeln(sprintf( + '↳ %s: validated %s, processed %s of %s (%s ok, %s warned, %s errored)', + $oImport->status->value, + $oImport->validated_count, + $oImport->processed_count, + $oImport->row_count ?? 0, + $oImport->success_count, + $oImport->warning_count, + $oImport->error_count + )); + + if ($this->fingerprint($oImport) === $sBefore) { + /** + * Recorded on the job as well as printed: console output + * from a cron run is nobody's idea of a diagnostic, and + * an admin with no shell has to be able to see why this + * stopped. + */ + $oImport = $oProcessor->abandon( + $oImport, + 'no progress was made between two consecutive attempts' + ); + + $this->oOutput->writeln('↳ No progress was made, abandoning this job'); + break; + } + } + + $this->reportOutcome($oImport); + + } finally { + $oModel->release($oImport->id); + } + } + + if (empty($iFound)) { + $this->oOutput->writeln('Nothing to do'); + } + } + + // -------------------------------------------------------------------------- + + /** + * Prints why a job ended the way it did + * + * The job's stored error already carries the phase, the cause and the first + * of the failing lines, so it is printed rather than recomposed here; that + * way the console, the details modal and the notification email can never + * disagree with one another. + */ + protected function reportOutcome(Resource\User\Import $oImport): void + { + if ($oImport->status === Status::FAILED) { + $this->error(array_merge( + [sprintf('Import #%s failed', $oImport->id)], + array_values(array_filter( + explode("\n", (string) $oImport->error), + fn(string $sLine): bool => trim($sLine) !== '' + )) + )); + + } elseif ($oImport->status === Status::PARTIAL) { + /** + * A warning is a row whose account exists but whose follow-up did + * not happen, so it is named separately: the reader's next step for + * an errored row is to correct the CSV, and for a warned one it is + * to finish the account by hand. + */ + $this->warning(array_values(array_filter([ + sprintf( + 'Import #%s finished with %s of %s rows unresolved', + $oImport->id, + $oImport->error_count + $oImport->warning_count, + $oImport->row_count ?? 0 + ), + $oImport->error_count + ? sprintf('%s rows failed and no account was created', $oImport->error_count) + : null, + $oImport->warning_count + ? sprintf( + '%s accounts were created but something afterwards did not complete', + $oImport->warning_count + ) + : null, + ]))); + } + + if ($oImport->log_id) { + $this->oOutput->writeln(sprintf( + '↳ Log: CDN object #%s', + $oImport->log_id + )); + } + } + + // -------------------------------------------------------------------------- + + /** + * Returns the oldest unclaimed job which this runner owns + * + * @param int[] $aSkip Jobs which have already been looked at this run + * + * @throws FactoryException + * @throws ModelException + */ + protected function getNextJob(array $aSkip): ?Resource\User\Import + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $oDb + ->select('id') + ->where('runner', Runner::CRON->value) + ->where_in('status', Status::values(Status::active())) + ->where('claim_token', null) + ->order_by('id', 'asc') + ->limit(1); + + if (!empty($aSkip)) { + $oDb->where_not_in('id', $aSkip); + } + + $oRow = $oDb->get($oModel->getTableName())->row(); + + /** @var Resource\User\Import|null $oImport */ + $oImport = $oRow ? $oModel->getById((int) $oRow->id) : null; + + return $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * A cheap representation of the job's progress, used to detect a stall + */ + protected function fingerprint(Resource\User\Import $oImport): string + { + return implode(':', [ + $oImport->status->value, + $oImport->validated_count, + $oImport->processed_count, + ]); + } +} diff --git a/src/Controller/Base.php b/src/Controller/Base.php index f39deef9..b73d809b 100644 --- a/src/Controller/Base.php +++ b/src/Controller/Base.php @@ -28,7 +28,7 @@ public function __construct() parent::__construct(); $oConfig = Factory::service('Config'); $oConfig->load('auth/auth'); - get_instance()->lang->load('auth/auth'); + Factory::service('Translation')->load('auth'); } // -------------------------------------------------------------------------- diff --git a/src/Cron/Task/User/Import/Clean.php b/src/Cron/Task/User/Import/Clean.php new file mode 100644 index 00000000..6aec2a25 --- /dev/null +++ b/src/Cron/Task/User/Import/Clean.php @@ -0,0 +1,39 @@ +query(<<query(<< $oStatus->value, $aStatuses); + } +} diff --git a/src/Event/Listener/User/Init.php b/src/Event/Listener/User/Init.php index 99540a71..873f9601 100644 --- a/src/Event/Listener/User/Init.php +++ b/src/Event/Listener/User/Init.php @@ -151,7 +151,7 @@ protected function checkUserIsSuspended(): self /** @var Authentication $oAuthService */ $oAuthService = Factory::service('Authentication', Constants::MODULE_SLUG); - get_instance()->lang->load('auth/auth'); + Factory::service('Translation')->load('auth'); $oAuthService->logout(); diff --git a/src/Exception/User/Import/LogException.php b/src/Exception/User/Import/LogException.php new file mode 100644 index 00000000..39cb75e6 --- /dev/null +++ b/src/Exception/User/Import/LogException.php @@ -0,0 +1,19 @@ +type('user_import_complete'); + } + + // -------------------------------------------------------------------------- + + /** + * Returns test data to use when sending test emails + * + * @return array + */ + public function getTestData(): array + { + return [ + 'import' => [ + 'row_count' => 100, + 'success' => 97, + 'warnings' => 1, + 'errors' => 2, + 'url' => siteUrl('admin/auth/import/preview/1'), + ], + /** + * Null on the happy path; complete() only writes an error here when + * the log itself could not be attached, or when the app's + * onImportComplete() hook objected. One block, so no phase line - + * see Service\User\Import\Processor::summariseError(). + */ + 'error' => null, + 'log_url' => siteUrl('admin/auth/import/log/1'), + ]; + } +} diff --git a/src/Factory/Email/User/Import/Failed.php b/src/Factory/Email/User/Import/Failed.php new file mode 100644 index 00000000..8bdfbfa7 --- /dev/null +++ b/src/Factory/Email/User/Import/Failed.php @@ -0,0 +1,51 @@ +type('user_import_failed'); + } + + // -------------------------------------------------------------------------- + + /** + * Returns test data to use when sending test emails + * + * @return array + */ + public function getTestData(): array + { + return [ + 'import' => [ + 'warnings' => 0, + 'errors' => 3, + 'url' => siteUrl('admin/auth/import/preview/1'), + ], + /** + * Summary and phase, which is all summariseError() lets through - + * the quoted rows and the cause stay in the log. Two blocks, because + * that is the shape the template has to be previewed in. + */ + 'error' => implode("\n\n", [ + '3 rows in the CSV could not be validated, so no user accounts were created. Correct them and upload the file again.', + 'Phase: validating the CSV (100 of 100 rows checked)', + ]), + 'log_url' => siteUrl('admin/auth/import/log/1'), + ]; + } +} diff --git a/src/Model/User/Import.php b/src/Model/User/Import.php new file mode 100644 index 00000000..7d8d13e6 --- /dev/null +++ b/src/Model/User/Import.php @@ -0,0 +1,312 @@ +hasOne('user', 'User', Constants::MODULE_SLUG, 'created_by') + ->hasOne('object', 'Object', Cdn\Constants::MODULE_SLUG) + ->hasOne('log', 'Object', Cdn\Constants::MODULE_SLUG); + } + + // -------------------------------------------------------------------------- + + /** + * Attempts to take exclusive ownership of a job + * + * The update only lands if the job is unclaimed and in a status a runner is + * permitted to pick up; the read back confirms it was this process which won. + * + * @param int $iId The job to claim + * @param string $sToken The token to claim it with + * + * @throws FactoryException + * @throws ModelException + */ + public function claim(int $iId, string $sToken): bool + { + $oDb = $this->getDb(); + + $oDb + ->set('claim_token', $sToken) + ->set('claimed', 'NOW()', false) + ->where('id', $iId) + ->where('claim_token', null) + ->where_in('status', Status::values(Status::active())) + ->update($this->getTableName()); + + return (bool) $oDb + ->where('id', $iId) + ->where('claim_token', $sToken) + ->count_all_results($this->getTableName()); + } + + // -------------------------------------------------------------------------- + + /** + * Relinquishes ownership of a job + * + * @throws FactoryException + * @throws ModelException + */ + public function release(int $iId): void + { + $oDb = $this->getDb(); + + $oDb + ->set('claim_token', null) + ->set('claimed', null) + ->where('id', $iId) + ->update($this->getTableName()); + } + + // -------------------------------------------------------------------------- + + /** + * Records how far through the validation phase the job is + * + * @throws FactoryException + * @throws ModelException + */ + public function setValidatedCount(int $iId, int $iCount): void + { + if (!$this->update($iId, ['validated_count' => $iCount])) { + throw new ModelException($this->lastError()); + } + } + + // -------------------------------------------------------------------------- + + /** + * Recalculates the error count from the item table + * + * The counts are derived rather than incremented so that a job which is + * interrupted and resumed cannot double count, or lose, a row. + * + * @throws FactoryException + * @throws ModelException + */ + public function syncValidationCounts(int $iId): void + { + $this->update($iId, [ + 'error_count' => $this->countItems($iId, ItemStatus::ERROR), + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Recalculates the processing cursor and counts from the item table + * + * @throws FactoryException + * @throws ModelException + */ + public function syncProcessingCounts(int $iId): void + { + $this->update($iId, [ + 'processed_count' => $this->countItems($iId), + 'success_count' => $this->countItems($iId, ItemStatus::SUCCESS), + 'warning_count' => $this->countItems($iId, ItemStatus::WARNING), + 'error_count' => $this->countItems($iId, ItemStatus::ERROR), + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Counts the job's items, optionally of a given status + * + * @throws FactoryException + * @throws ModelException + */ + protected function countItems(int $iId, ?ItemStatus $oStatus = null): int + { + /** @var Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + $oDb = $this->getDb(); + $oDb->where('import_id', $iId); + + if ($oStatus !== null) { + $oDb->where('status', $oStatus->value); + } + + return (int) $oDb->count_all_results($oItemModel->getTableName()); + } + + // -------------------------------------------------------------------------- + + /** + * Destroys the CDN objects belonging to a job + * + * Call this only *after* the job row has been deleted: `object_id` is + * ON DELETE CASCADE, so destroying the CSV while the row still exists takes + * the row with it - inside objectDestroy()'s own transaction - and the + * delete() which follows then reports the job as missing. + * + * Failures are collected rather than thrown: the row is already gone, so a + * file which outlives it is litter, not an error. Note that + * `auth:user:import:clean` walks job rows, so it will never reap these. + * objectDestroy() signals failure both by returning false - for a missing + * object, a driver failure, or a rolled back transaction - and by throwing, + * so both are handled. + * + * @return array The reason each object could not be destroyed, keyed by object ID + * + * @throws FactoryException + */ + public function destroyObjects(Resource\User\Import $oImport): array + { + $oCdn = $this->getCdn(); + $aErrors = []; + + foreach (array_filter([$oImport?->object?->id ?? $oImport->object_id, $oImport->log_id]) as $iObjectId) { + try { + if (!$oCdn->objectDestroy($iObjectId)) { + $aErrors[(int) $iObjectId] = $oCdn->lastError() ?: 'Unknown error'; + } + } catch (Throwable $e) { + $aErrors[(int) $iObjectId] = $e->getMessage(); + } + } + + return $aErrors; + } + + // -------------------------------------------------------------------------- + + /** + * Deletes the job's items + * + * Called when a job is restarted from PENDING. The unique key on + * (import_id, line) is what makes resuming idempotent, so leaving the + * previous run's items in place would make every line look as though it had + * already been handled - and the counts, which are re-derived rather than + * incremented, would be taken from that stale run. + * + * @return bool Whether the items were deleted + * @throws FactoryException + */ + public function deleteItems(int $iId): bool + { + /** @var Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + return (bool) $this + ->getDb() + ->where('import_id', $iId) + ->delete($oItemModel->getTableName()); + } + + // -------------------------------------------------------------------------- + + /** + * Returns the database service + * + * Broken out so that the claim/release contract can be exercised in + * isolation, without standing up a database. + * + * @throws FactoryException + */ + protected function getDb(): Database + { + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + return $oDb; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the CDN service + * + * Broken out so that destroyObjects() can be exercised in isolation, as + * getDb() does for the database. + * + * @throws FactoryException + */ + protected function getCdn(): Cdn\Service\Cdn + { + /** @var Cdn\Service\Cdn $oCdn */ + $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + return $oCdn; + } +} diff --git a/src/Model/User/Import/Item.php b/src/Model/User/Import/Item.php new file mode 100644 index 00000000..470e087f --- /dev/null +++ b/src/Model/User/Import/Item.php @@ -0,0 +1,79 @@ +hasOne('user', 'User', Constants::MODULE_SLUG); + } +} diff --git a/src/Queue/Task/User/Import.php b/src/Queue/Task/User/Import.php new file mode 100644 index 00000000..667140f2 --- /dev/null +++ b/src/Queue/Task/User/Import.php @@ -0,0 +1,130 @@ +get(); + + /** @var Resource\User\Import|null $oImport */ + $oImport = $oModel->getById($iId); + + if (empty($oImport)) { + throw new RuntimeException(sprintf( + 'User import #%s does not exist', + $iId + )); + + } elseif ($oImport->status->isTerminal()) { + return; + } + + $sToken = md5(uniqid((string) getmypid(), true)); + + if (!$oModel->claim($iId, $sToken)) { + // Another process is already working this job + return; + } + + try { + + while (!$oImport->status->isTerminal()) { + + $sBefore = $this->fingerprint($oImport); + $oImport = $oProcessor->process($oImport); + + if ($this->fingerprint($oImport) === $sBefore) { + + /** + * Recorded on the job before it is thrown: module-queue's own + * failure record is not reachable from the user import admin, + * so without this an admin sees a job stuck mid-flight with + * nothing to explain it. + */ + $oProcessor->abandon( + $oImport, + 'no progress was made between two consecutive attempts' + ); + + throw new RuntimeException(sprintf( + 'User import #%s made no progress; abandoning to avoid spinning', + $iId + )); + } + } + + } finally { + $oModel->release($iId); + } + } + + // -------------------------------------------------------------------------- + + /** + * A cheap representation of the job's progress, used to detect a stall + */ + protected function fingerprint(Resource\User\Import $oImport): string + { + return implode(':', [ + $oImport->status->value, + $oImport->validated_count, + $oImport->processed_count, + ]); + } +} diff --git a/src/Resource/User/Import.php b/src/Resource/User/Import.php new file mode 100644 index 00000000..c106aaa6 --- /dev/null +++ b/src/Resource/User/Import.php @@ -0,0 +1,125 @@ +status = $resource->status instanceof Status + ? $resource->status + : Status::from($resource->status); + + $resource->runner = $resource->runner instanceof Runner || $resource->runner === null + ? $resource->runner + : Runner::tryFrom($resource->runner); + + if (!$resource->additional instanceof stdClass) { + $resource->additional = (object) (json_decode((string) $resource->additional) ?: []); + } + + $resource->skip_registered = (bool) ($resource->skip_registered ?? false); + $resource->warning_count = (int) ($resource->warning_count ?? 0); + + parent::__construct($resource, $model); + } + + // -------------------------------------------------------------------------- + + /** + * The percentage of the job which has been completed + */ + public function getPercent(): int + { + if (empty($this->row_count)) { + return 0; + + } elseif ($this->status->isTerminal()) { + return 100; + } + + $iCursor = $this->status === Status::VALIDATING + ? $this->validated_count + : $this->processed_count; + + return (int) min(100, floor(($iCursor / $this->row_count) * 100)); + } +} diff --git a/src/Resource/User/Import/Item.php b/src/Resource/User/Import/Item.php new file mode 100644 index 00000000..c0947ae9 --- /dev/null +++ b/src/Resource/User/Import/Item.php @@ -0,0 +1,33 @@ +status = $resource->status instanceof ItemStatus + ? $resource->status + : ItemStatus::from($resource->status); + + parent::__construct($resource, $model); + } +} diff --git a/src/Service/User/Import.php b/src/Service/User/Import.php index 1767ea05..1aeef8a5 100644 --- a/src/Service/User/Import.php +++ b/src/Service/User/Import.php @@ -14,13 +14,17 @@ use Closure; use Nails\Auth\Constants; +use Nails\Auth\Exception\User\Import\TemplateException; use Nails\Auth\Model\User; +use Nails\Auth\Resource; use Nails\Common\Exception\FactoryException; use Nails\Common\Exception\NailsException; use Nails\Common\Exception\ValidationException; use Nails\Common\Factory\Model\Field; +use Nails\Common\Service\Database; use Nails\Common\Service\DateTime; use Nails\Common\Service\FormValidation; +use Nails\Common\Validation\Context; use Nails\Config; use Nails\Factory; @@ -31,6 +35,21 @@ */ class Import { + /** + * The number of values whichExist() puts in a single IN clause + * + * CodeIgniter's query builder runs a preg_match over the whole compiled + * condition, so one enormous IN clause fails outright with "regular + * expression is too large" - a 1,362 value list compiles to roughly 35KB, + * past what PCRE will accept. Chunking also keeps the statement well inside + * max_allowed_packet. Still one query per few hundred values rather than one + * per row, which is the point. + * + * @var int + */ + const WHICH_EXIST_CHUNK_SIZE = 250; + + /** * Returns the columns which can be imported * @@ -67,6 +86,85 @@ public function getUniqueKeys(): array ]; } + /** + * Returns which of the given values are already taken + * + * Batched on purpose: this is asked once per unique key for a whole file, or + * a whole chunk, rather than once per row. One indexed lookup answers it, + * where the per-row closures this replaced cost a query each. + * + * An app which adds a unique key must handle it here; there is deliberately + * no silent fallback, because a uniqueness check which quietly answers "none + * of them" is worse than one which fails. The key is checked before the + * empty-value short circuit so an unsupported key is caught even when there + * is nothing to look up. + * + * @param string $sKey The unique key being checked + * @param string[] $aValues The values to look for + * + * @return string[] The values which are already taken, lowercased + * @throws TemplateException If the key cannot be looked up + * @throws FactoryException + */ + public function whichExist(string $sKey, array $aValues): array + { + /** @var User\Email $oUserEmailModel */ + $oUserEmailModel = Factory::model('UserEmail', Constants::MODULE_SLUG); + /** @var User $oUserModel */ + $oUserModel = Factory::model('User', Constants::MODULE_SLUG); + + [$sTable, $sColumn] = match ($sKey) { + 'email' => [$oUserEmailModel->getTableName(), 'email'], + 'username' => [$oUserModel->getTableName(), 'username'], + default => throw new TemplateException(sprintf( + '"%s" is listed by %s::getUniqueKeys() but %s::whichExist() does not know how to ' + . 'look it up; override it to say where the value lives.', + $sKey, + static::class, + static::class + )), + }; + + $aValues = array_values( + array_unique( + array_filter( + array_map(fn($mValue): string => strtolower(trim((string) $mValue)), $aValues), + fn(string $sValue): bool => $sValue !== '' + ) + ) + ); + + if (empty($aValues)) { + return []; + } + + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aExisting = []; + + foreach (array_chunk($aValues, static::WHICH_EXIST_CHUNK_SIZE) as $aChunk) { + + /** + * Queried directly rather than through the model: this is an + * existence probe, and Model\User::getCountCommon() would hydrate + * every match across three joins to answer a question about a single + * column. + */ + $aRows = $oDb + ->select($sColumn) + ->where_in($sColumn, $aChunk) + ->get($sTable) + ->result(); + + foreach ($aRows as $oRow) { + $aExisting[] = strtolower(trim((string) $oRow->{$sColumn})); + } + } + + return $aExisting; + } + /** * Returns the validation rules for a given key * @@ -85,36 +183,25 @@ public function getValidationRules(string $key): array $oUserPasswordModel = Factory::model('UserPassword', Constants::MODULE_SLUG); /** @var DateTime $oDateTimeService */ $oDateTimeService = Factory::service('DateTime'); - /** @var FormValidation $oFormValidationService */ - $oFormValidationService = Factory::service('FormValidation'); return match ($key) { + /** + * Note that uniqueness is deliberately absent from here and from + * 'username'. It used to be a closure calling getByEmail() per row, + * which cost a query - two, on a hit - for every line of the file. + * It is now a batched whole-of-file concern; see whichExist() and + * Import\Validator::detectRegistered(). + */ 'email' => array_filter([ in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['EMAIL', 'BOTH']) ? FormValidation::RULE_REQUIRED : null, FormValidation::RULE_VALID_EMAIL, - function ($email) use ($oUserModel) { - if ($email && $oUserModel->getByEmail($email)) { - throw new ValidationException(sprintf( - '"%s" is already registered', - $email - )); - } - }, ]), 'username' => array_filter([ in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['USERNAME', 'BOTH']) ? FormValidation::RULE_REQUIRED : null, - function ($username) use ($oUserModel) { - if ($username && $oUserModel->getByUsername($username)) { - throw new ValidationException(sprintf( - '"%s" is already registered', - $username - )); - } - }, function ($username) use ($oUserModel) { if ($username && !$oUserModel->isValidUsername($username)) { throw new ValidationException(sprintf( @@ -138,13 +225,13 @@ function ($groupId) use ($oUserGroupModel) { }, ], 'password' => [ - function ($password) use ($oUserPasswordModel, $oUserGroupModel, $oFormValidationService) { + function ($password, Context $oContext) use ($oUserPasswordModel, $oUserGroupModel) { if (!$password) { return; } - $groupId = $oFormValidationService->validation_data['group_id'] ?? null; + $groupId = $oContext->getValue('group_id'); $group = $oUserGroupModel->getById((int) $groupId) ?? $oUserGroupModel->getDefaultGroup(); if (!$oUserPasswordModel->isAcceptable($group, $password)) { @@ -254,4 +341,122 @@ public function parseAdditionalFields(string $sKey, mixed $mValue): mixed { return $mValue; } + + // -------------------------------------------------------------------------- + // Lifecycle hooks + // + // All five are no-ops here, and an override should call parent:: as + // whichExist() documents. Read the failure semantics before overriding: + // they are not the same for each hook, because what the processor can still + // do about a failure is not the same at each point. + // -------------------------------------------------------------------------- + + /** + * Fired once, when validation has passed and before the first account is created + * + * The one hook which may refuse the whole job: this runs while nothing has + * been created, so a throw here is deliberately left to reach the processor, + * which fails the job with the reason folded into its composed error. Throw + * a ValidationException with a sentence an admin can act on. + * + * Note that it fires once per job, not once per chunk - run() is called + * many times, across processes, but this is not. + * + * @param Resource\User\Import $oImport The job which is about to start + */ + public function onImportStart(Resource\User\Import $oImport): void + { + } + + // -------------------------------------------------------------------------- + + /** + * Fired for every row, immediately before the account is created + * + * The data is returned rather than taken by reference; whatever comes back + * is what reaches Model\User::create(), so an override which forgets to + * return imports nothing. Keys create() does not recognise - anything which + * is neither a user column nor a meta field - are silently discarded by it, + * so app-specific values belong in afterUserCreate() rather than here. + * + * This runs *after* the job's additional fields have been applied, so an + * override sees, and may overrule, what parseAdditionalFields() decided. + * + * A throw marks the row ERROR and leaves no account behind; the rest of the + * job carries on. + * + * @param array $aUserData The data create() will be given + * @param Resource\User\Import $oImport The job the row belongs to + * @param array $aRow The CSV row, aligned to the header, + * including columns create() will drop + * + * @return array The data to create the account with + */ + public function prepareUserData(array $aUserData, Resource\User\Import $oImport, array $aRow): array + { + return $aUserData; + } + + // -------------------------------------------------------------------------- + + /** + * Fired for every row, immediately after the account is created + * + * This is where app-specific follow-on work belongs - applying data a user + * column cannot hold, starting a subscription, and so on. + * + * The account is committed by the time this is called: Model\User::create() + * commits, fires USER_CREATED and queues the welcome email before it + * returns, so nothing here can undo it. A throw is therefore recorded + * rather than retried - the row is marked WARNING against the new account's + * ID, the exception is reported, and the job finishes PARTIAL so that a + * silent failure cannot pass as a clean import. + * + * @param Resource\User $oUser The account which was just created + * @param array $aUserData The data it was created with + * @param Resource\User\Import $oImport The job the row belongs to + */ + public function afterUserCreate( + Resource\User $oUser, + array $aUserData, + Resource\User\Import $oImport + ): void { + } + + // -------------------------------------------------------------------------- + + /** + * Fired once, when the job has finished + * + * Every account which was going to be created already exists, so this must + * not pretend the job can be undone. A throw is reported and appended to the + * job's stored error - which is what the details modal and the notification + * email show - but it cannot re-brand the job: a finished import stays + * COMPLETE or PARTIAL. + * + * @param Resource\User\Import $oImport The job, as it now stands + */ + public function onImportComplete(Resource\User\Import $oImport): void + { + } + + // -------------------------------------------------------------------------- + + /** + * Fired once, when the job has been rejected or abandoned + * + * Note that this fires for a job rejected during validation, where no + * account was ever created, as well as for one brought down part way + * through, where some exist; read `$oImport->success_count` rather than + * assuming either. + * + * A throw is reported and otherwise swallowed. It cannot be rethrown: the + * processor's failure path is reached from its own catch-all, so an + * exception escaping here would recurse straight back into it. + * + * @param Resource\User\Import $oImport The job, as it now stands + */ + public function onImportFailed(Resource\User\Import $oImport): void + { + } } diff --git a/src/Service/User/Import/Csv.php b/src/Service/User/Import/Csv.php new file mode 100644 index 00000000..c09cefa8 --- /dev/null +++ b/src/Service/User/Import/Csv.php @@ -0,0 +1,358 @@ +open($sPath); + + try { + $aHeader = $this->read($rHandle); + } finally { + fclose($rHandle); + } + + if ($this->isBlank($aHeader)) { + return []; + } + + return array_map(fn($sValue) => trim((string) $sValue), $aHeader); + } + + // -------------------------------------------------------------------------- + + /** + * Counts the number of data rows in the CSV, the header excluded + * + * @param string $sPath The path to the CSV + */ + public function countRows(string $sPath): int + { + $rHandle = $this->open($sPath); + + try { + + // Discard the header + $this->read($rHandle); + + $iCount = 0; + while (($aRow = $this->read($rHandle)) !== false) { + if (!$this->isBlank($aRow)) { + $iCount++; + } + } + + } finally { + fclose($rHandle); + } + + return $iCount; + } + + // -------------------------------------------------------------------------- + + /** + * Yields raw (i.e. unkeyed) data rows, keyed by their line number + * + * @param string $sPath The path to the CSV + * @param int $iOffset The number of data rows to skip + * @param int|null $iLimit The maximum number of rows to yield, null for all + * + * @return Generator + */ + public function readRawRows(string $sPath, int $iOffset = 0, ?int $iLimit = null): Generator + { + $rHandle = $this->open($sPath); + + try { + + // Discard the header + $this->read($rHandle); + + $iIndex = 0; + $iSent = 0; + + while (($aRow = $this->read($rHandle)) !== false) { + + if ($this->isBlank($aRow)) { + continue; + } + + $iIndex++; + + if ($iIndex <= $iOffset) { + continue; + + } elseif ($iLimit !== null && $iSent >= $iLimit) { + break; + } + + $iSent++; + + yield $iIndex + static::HEADER_LINE => $aRow; + } + + } finally { + fclose($rHandle); + } + } + + // -------------------------------------------------------------------------- + + /** + * Yields data rows keyed by line number, each combined with the header and + * with the import service's default values applied to blank cells + * + * @param string $sPath The path to the CSV + * @param int $iOffset The number of data rows to skip + * @param int|null $iLimit The maximum number of rows to yield, null for all + * + * @return Generator> + * @throws FactoryException + * @throws NailsException + */ + public function readRows(string $sPath, int $iOffset = 0, ?int $iLimit = null): Generator + { + $aHeader = $this->getHeader($sPath); + + foreach ($this->readRawRows($sPath, $iOffset, $iLimit) as $iLine => $aRow) { + yield $iLine => $this->applyDefaults( + $this->align($aHeader, $aRow) + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Keys a raw row by the header's columns + * + * Rows which are short are padded, rows which are long are truncated; the + * discrepancy is reported separately by the validator so that a malformed + * row produces a legible error rather than an exception. + * + * @param string[] $aHeader The CSV's header row + * @param string[] $aRow The raw row + * + * @return array + */ + public function align(array $aHeader, array $aRow): array + { + $aOut = []; + $aRow = array_values($aRow); + $iIndex = 0; + + foreach ($aHeader as $sKey) { + $aOut[$sKey] = trim((string) ($aRow[$iIndex] ?? '')); + $iIndex++; + } + + return $aOut; + } + + // -------------------------------------------------------------------------- + + /** + * Replaces blank cells with the import service's default value for that key + * + * @param array $aRow The keyed row + * + * @return array + * @throws FactoryException + * @throws NailsException + */ + public function applyDefaults(array $aRow): array + { + /** @var \Nails\Auth\Service\User\Import $oImportService */ + $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); + + foreach ($aRow as $sKey => $sValue) { + if ($sValue === '') { + $aRow[$sKey] = $oImportService->getDefaultValue($sKey); + } + } + + return $aRow; + } + + // -------------------------------------------------------------------------- + + /** + * Builds the import's log CSV + * + * The source CSV is streamed and joined, line by line, against the supplied + * items; this reproduces the log format the import has always emitted (the + * original columns, plus `id`, `status` and `message`) without ever having + * held a second copy of the file. + * + * @param string $sPath The path to the source CSV + * @param iterable $aItems Keyed by line + * number, ascending + * + * @return string The path to the generated log CSV + */ + public function writeLog(string $sPath, iterable $aItems): string + { + $oItems = $aItems instanceof Iterator + ? $aItems + : new \ArrayIterator(is_array($aItems) ? $aItems : iterator_to_array($aItems)); + + $oItems->rewind(); + + $aHeader = $this->getHeader($sPath); + $sLog = $this->getTempFile(); + $rLog = fopen($sLog, 'w'); + + if ($rLog === false) { + throw new RuntimeException(sprintf('Failed to open "%s" for writing', $sLog)); + } + + try { + + fputcsv($rLog, array_merge($aHeader, ['id', 'status', 'message']), escape: ''); + + foreach ($this->readRawRows($sPath) as $iLine => $aRow) { + + if (!$oItems->valid()) { + break; + + } elseif ((int) $oItems->key() !== $iLine) { + continue; + } + + $aItem = (array) $oItems->current(); + + fputcsv( + $rLog, + array_merge( + array_values($this->align($aHeader, $aRow)), + [ + $aItem['id'] ?? null, + $aItem['status'] ?? '', + $aItem['message'] ?? '', + ] + ), + escape: '' + ); + + $oItems->next(); + } + + } finally { + fclose($rLog); + } + + return $sLog; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the path to a new temporary file + */ + public function getTempFile(): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-user-import-'); + + if ($sPath === false) { + throw new RuntimeException('Failed to create a temporary file'); + } + + return $sPath; + } + + // -------------------------------------------------------------------------- + + /** + * Opens the CSV for reading + * + * @return resource + */ + protected function open(string $sPath) + { + $rHandle = @fopen($sPath, 'r'); + + if ($rHandle === false) { + throw new RuntimeException(sprintf( + 'Failed to open "%s" for reading', + $sPath + )); + } + + return $rHandle; + } + + // -------------------------------------------------------------------------- + + /** + * Reads a single record + * + * The empty escape character matches the `str_getcsv($line, escape: '')` + * semantics the import has always used; `SplFileObject` cannot be relied + * upon to honour it, hence the plain handle. + * + * @param resource $rHandle + * + * @return string[]|false + */ + protected function read($rHandle): array|false + { + return fgetcsv($rHandle, escape: ''); + } + + // -------------------------------------------------------------------------- + + /** + * Whether a record is an empty line + * + * @param string[]|false $aRow + */ + protected function isBlank(array|false $aRow): bool + { + return $aRow === false || $aRow === [null] || $aRow === ['']; + } +} diff --git a/src/Service/User/Import/Dispatcher.php b/src/Service/User/Import/Dispatcher.php new file mode 100644 index 00000000..0e23d919 --- /dev/null +++ b/src/Service/User/Import/Dispatcher.php @@ -0,0 +1,122 @@ +getRunner(); + + if (!$oModel->update($oImport->id, [ + 'runner' => $oRunner->value, + 'status' => Status::PENDING->value, + ])) { + throw new ModelException($oModel->lastError()); + } + + if ($oRunner === Runner::QUEUE) { + Factory::service('Manager', \Nails\Queue\Constants::MODULE_SLUG) + ->push( + new ImportTask(), + Factory::factory('Data', \Nails\Queue\Constants::MODULE_SLUG, $oImport->id) + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Whether the runner has checked in recently enough to be believed + * + * There is nothing to check for the queue runner; module-queue monitors its + * own workers, and a warning we cannot substantiate is worse than none. + * + * @throws FactoryException + */ + public function isRunning(): bool + { + if ($this->getRunner() === Runner::QUEUE) { + return true; + } + + $sLastRun = appSetting('user-import-cron-last-run', Constants::MODULE_SLUG); + if (empty($sLastRun)) { + return false; + } + + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + return ($oNow->getTimestamp() - (new \DateTime($sLastRun))->getTimestamp()) <= static::RUNNING_THRESHOLD; + } + + // -------------------------------------------------------------------------- + + /** + * Which runner will process jobs on this installation + */ + public function getRunner(): Runner + { + return Components::exists(static::QUEUE_MODULE) + ? Runner::QUEUE + : Runner::CRON; + } +} diff --git a/src/Service/User/Import/Processor.php b/src/Service/User/Import/Processor.php new file mode 100644 index 00000000..fd75d25b --- /dev/null +++ b/src/Service/User/Import/Processor.php @@ -0,0 +1,1674 @@ + 'import-user', + 'is_hidden' => true, + 'allowed_types' => 'csv', + ]; + + /** + * The number of rows to handle in a single call to process() + * + * @var int + */ + const CHUNK_SIZE = 100; + + /** + * The number of items to read from the database at a time when building the log + * + * @var int + */ + const LOG_PAGE_SIZE = 1000; + + /** + * The number of failing lines to quote in the job's error + * + * Enough to recognise a pattern - "they are all already registered" - without + * duplicating the log CSV, which carries every row. + * + * @var int + */ + const ERROR_SAMPLE_SIZE = 10; + + /** + * The prefix of the composed error's "how far did it get" fact + * + * A const because summariseError() has to recognise this line to keep it + * while dropping the facts either side of it; the two must not drift. + * + * @var string + */ + const ERROR_FACT_PHASE = 'Phase: '; + + /** + * The maximum length of the job's error + * + * The column is TEXT, so this is a sanity bound rather than a hard limit; it + * exists so a pathological CSV cannot fill the row with error text. + * + * @var int + */ + const ERROR_MAX_LENGTH = 8000; + + /** + * The maximum length of a stack trace recorded in the log + * + * @var int + */ + const ERROR_TRACE_MAX_LENGTH = 2000; + + /** + * The levels report() can log at + * + * @var string + */ + const LOG_INFO = 'info'; + const LOG_WARNING = 'warning'; + const LOG_ERROR = 'error'; + + // -------------------------------------------------------------------------- + + /** + * Advances the job by, at most, $iLimit rows + * + * @param Resource\User\Import $oImport The job to advance + * @param int|null $iLimit The number of rows to handle, null for the configured chunk size + * + * @return Resource\User\Import The job, as it now stands + * @throws FactoryException + * @throws ModelException + * @throws NailsException + */ + public function process(Resource\User\Import $oImport, ?int $iLimit = null): Resource\User\Import + { + $iLimit = $iLimit ?: static::getChunkSize(); + + try { + + return match ($oImport->status) { + Status::PENDING => $this->begin($oImport), + Status::VALIDATING => $this->validate($oImport, $iLimit), + Status::RUNNING => $this->run($oImport, $iLimit), + default => $oImport, + }; + + } catch (Throwable $e) { + /** + * Re-read before composing: the counts the message quotes have to be + * the ones on disk, not the ones this call started with. + */ + $oImport = $this->refresh($oImport); + + return $this->fail($oImport, $this->composeThrowableError($oImport, $e), $e); + } + } + + // -------------------------------------------------------------------------- + + /** + * Rejects a job which has stopped making progress + * + * Public because it is the runners, not the processor, which detect a stall - + * they are the ones holding the before-and-after fingerprints. Named to match + * the copy they already print rather than exposing fail() itself. + * + * @throws FactoryException + * @throws ModelException + */ + public function abandon(Resource\User\Import $oImport, string $sReason): Resource\User\Import + { + $oImport = $this->refresh($oImport); + + return $this->fail($oImport, $this->composeStallError($oImport, $sReason)); + } + + // -------------------------------------------------------------------------- + + /** + * The number of rows to handle in a single call to process() + */ + public static function getChunkSize(): int + { + return (int) Config::get('AUTH_USER_IMPORT_CHUNK', static::CHUNK_SIZE) ?: static::CHUNK_SIZE; + } + + // -------------------------------------------------------------------------- + + /** + * Opens the job: records when it started and how much work there is to do + * + * @throws FactoryException + * @throws ModelException + */ + protected function begin(Resource\User\Import $oImport): Resource\User\Import + { + $oModel = $this->getModel(); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + $sPath = $this->getSourcePath($oImport); + + $this->clearPreviousAttempt($oImport); + + $iRowCount = $oCsv->countRows($sPath); + + $oModel->update($oImport->id, [ + 'started' => $oNow->format('Y-m-d H:i:s'), + 'finished' => null, + 'log_id' => null, + 'row_count' => $iRowCount, + 'validated_count' => 0, + 'processed_count' => 0, + 'success_count' => 0, + 'warning_count' => 0, + 'error_count' => 0, + 'error' => null, + 'status' => Status::VALIDATING->value, + ]); + + $this->report( + $oImport, + sprintf( + 'starting; %s rows, %s runner', + number_format($iRowCount), + $oImport->runner?->value ?? 'unknown' + ), + null, + static::LOG_INFO + ); + + return $this->refresh($oImport); + } + + // -------------------------------------------------------------------------- + + /** + * Validates the next chunk of rows + * + * Nothing is created during this phase. If a single row fails then the whole + * job is rejected, because a half-imported file is worse than no import at + * all; the errors are written out as a log so they can be corrected and the + * file re-uploaded. + * + * @throws FactoryException + * @throws ModelException + * @throws NailsException + */ + protected function validate(Resource\User\Import $oImport, int $iLimit): Resource\User\Import + { + $oModel = $this->getModel(); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + + $sPath = $this->getSourcePath($oImport); + $aHeader = $oCsv->getHeader($sPath); + + /** + * Both are fatal and neither produces a per-row log worth generating: a + * template which cannot identify an account, or a header we do not + * understand, means no row could ever be imported. + */ + $oValidator->validateTemplate(); + $oValidator->validateHeader($aHeader); + + $aChunk = iterator_to_array( + $oCsv->readRawRows($sPath, $oImport->validated_count, $iLimit), + true + ); + + $aErrors = $oValidator->validateRows($aHeader, $aChunk); + + /** + * Whether an already-registered row is fatal is the admin's call, taken + * when the job was approved. When they have opted to skip, the rows are + * deliberately *not* recorded here and are left for run() to deal with + * as it reaches them - see the note there on why an item must not exist + * ahead of its row. + */ + if (!$oImport->skip_registered) { + foreach ($oValidator->detectRegistered($aHeader, $aChunk) as $iLine => $aLineErrors) { + $aErrors[$iLine] = array_merge($aErrors[$iLine] ?? [], $aLineErrors); + } + } + + foreach ($aErrors as $iLine => $aLineErrors) { + $this->recordItem( + $oImport, + $iLine, + ItemStatus::ERROR, + implode('; ', $aLineErrors) + ); + } + + $oModel->setValidatedCount($oImport->id, $oImport->validated_count + count($aChunk)); + $oModel->syncValidationCounts($oImport->id); + + $oImport = $this->refresh($oImport); + + // More to do? + if (!empty($aChunk) && $oImport->validated_count < $oImport->row_count) { + return $oImport; + } + + if ($oImport->error_count) { + + $this->report( + $oImport, + sprintf( + 'validation rejected %s of %s rows', + number_format($oImport->error_count), + number_format((int) $oImport->row_count) + ), + null, + static::LOG_WARNING + ); + + return $this->fail($oImport, $this->composeValidationError($oImport)); + } + + /** + * The app's last chance to refuse the job, and the only hook which is + * allowed to: nothing has been created yet, so a throw here is + * deliberately left to reach process()'s catch, which fails the job with + * the reason folded into its composed error. Every other hook is + * wrapped, because by the time they run there are accounts which cannot + * be taken back. + */ + $this->getImportService()->onImportStart($oImport); + + $oModel->update($oImport->id, [ + 'processed_count' => 0, + 'success_count' => 0, + 'warning_count' => 0, + 'error_count' => 0, + 'status' => Status::RUNNING->value, + ]); + + return $this->refresh($oImport); + } + + // -------------------------------------------------------------------------- + + /** + * Creates the users for the next chunk of rows + * + * @throws FactoryException + * @throws ModelException + * @throws NailsException + */ + protected function run(Resource\User\Import $oImport, int $iLimit): Resource\User\Import + { + $oModel = $this->getModel(); + $oUserModel = $this->getUserModel(); + $oImportService = $this->getImportService(); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var Validator $oValidator */ + $oValidator = Factory::service('UserImportValidator', Constants::MODULE_SLUG); + + $sPath = $this->getSourcePath($oImport); + $aHeader = $oCsv->getHeader($sPath); + $aKeys = $oImportService->getKeys(); + + $aChunk = iterator_to_array( + $oCsv->readRawRows($sPath, $oImport->processed_count, $iLimit), + true + ); + + /** + * One batched lookup for the chunk, and only when the admin asked for it: + * without their consent a row which became registered since validation + * is left to create(), which refuses it on its own authority and reports + * its own reason. + */ + $aRegistered = $oImport->skip_registered + ? $oValidator->detectRegistered($aHeader, $aChunk) + : []; + + /** + * A job which was interrupted mid-chunk will have a cursor which lags the + * item table; skip anything already accounted for so a row can never be + * imported twice. + */ + $aDone = $this->getHandledLines($oImport, array_keys($aChunk)); + + foreach ($aChunk as $iLine => $aRow) { + + if (in_array($iLine, $aDone, true)) { + continue; + } + + $aDatum = $oCsv->align($aHeader, $aRow); + + /** + * Claim the line before doing anything which cannot be undone; if the + * process dies part way through, the line is marked as handled and + * will not be attempted again. + */ + $iItemId = $this->recordItem( + $oImport, + $iLine, + ItemStatus::ERROR, + 'The import was interrupted before this row was completed' + ); + + if (empty($iItemId)) { + continue; + } + + /** + * Claimed, then resolved without creating anything. Handling the skip + * here rather than pre-recording it during validation is deliberate: + * the read offset above is processed_count, which only tracks the + * file because every row of a chunk yields exactly one item. An item + * which exists ahead of its row inflates the count and silently + * steps over a row which was never imported. + */ + if (array_key_exists($iLine, $aRegistered)) { + $this->resolveItem( + $iItemId, + ItemStatus::ERROR, + sprintf('%s; skipped', implode('; ', $aRegistered[$iLine])) + ); + continue; + } + + $bSendEmail = stringToBoolean($aDatum['send_email'] ?? false); + $aUserData = []; + + foreach ($aKeys as $sKey) { + $aUserData[$sKey] = $aDatum[$sKey] ?? null; + if ($aUserData[$sKey] === '') { + $aUserData[$sKey] = $oImportService->getDefaultValue($sKey); + } + } + + // Apply additional fields + foreach ($oImport->additional as $sProperty => $mValue) { + $aUserData[$sProperty] = $oImportService->parseAdditionalFields($sProperty, $mValue); + } + + try { + + /** + * Deliberately after the loop above rather than in place of it, + * so an app which overrides the hook without calling parent:: + * still gets the additional fields it configured. The raw row is + * handed over too, because it carries the columns create() is + * about to discard. + */ + $aUserData = $oImportService->prepareUserData($aUserData, $oImport, $aDatum); + + $oUser = $oUserModel->create($aUserData, $bSendEmail); + + if (!$oUser) { + $this->resolveItem( + $iItemId, + ItemStatus::ERROR, + /** + * lastError() returns false, not '', for an empty error + * stack; without the fallback a failed create() leaves + * the row with no reason recorded against it at all. + */ + $oUserModel->lastError() ?: 'The account could not be created' + ); + continue; + } + + /** + * The account is committed from here on - create() commits, and + * queues the welcome email, before it returns - so nothing below + * may pretend it can be undone. A hook which fails leaves the row + * WARNING against the new account's ID, which is the thread back + * to whatever an admin now has to finish by hand. + */ + try { + $oImportService->afterUserCreate($oUser, $aUserData, $oImport); + $this->resolveItem($iItemId, ItemStatus::SUCCESS, null, $oUser->id); + + } catch (Throwable $e) { + $this->resolveItem( + $iItemId, + ItemStatus::WARNING, + sprintf('The account was created, but %s', $e->getMessage()), + $oUser->id + ); + $this->report( + $oImport, + sprintf('line %s: afterUserCreate() failed', $iLine), + $e + ); + } + + } catch (Throwable $e) { + // Covers prepareUserData() and create() itself; no account exists + $this->resolveItem($iItemId, ItemStatus::ERROR, $e->getMessage()); + $this->report($oImport, sprintf('line %s could not be imported', $iLine), $e); + } + } + + $oModel->syncProcessingCounts($oImport->id); + $oImport = $this->refresh($oImport); + + if (empty($aChunk) || $oImport->processed_count >= $oImport->row_count) { + return $this->complete($oImport); + } + + return $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * Finishes the job, generating and attaching the log + * + * @throws FactoryException + * @throws ModelException + * @throws NailsException + */ + protected function complete(Resource\User\Import $oImport): Resource\User\Import + { + $oModel = $this->getModel(); + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + $aLog = $this->attachLog($oImport); + + // A warned account exists, so it is one of the accounts this created + $iCreated = $oImport->success_count + $oImport->warning_count; + + /** + * A log which could not be written must not turn a finished import into + * a failed one - the accounts exist either way - but nor should it + * disappear, so it is recorded against the job. This used to be an + * un-caught call, which meant a log failure here reached process()'s + * catch and re-branded a perfectly good import as FAILED. + */ + $sError = $aLog['error'] + ? $this->truncateError($this->appendLogFailure( + sprintf( + '%s user %s created; the import itself finished normally.', + number_format($iCreated), + $iCreated === 1 ? 'account was' : 'accounts were' + ), + $aLog['error'] + )) + : null; + + /** + * A warning is a row whose account exists but whose follow-up did not + * happen, so it counts against a clean finish just as an error does; a + * job which quietly failed every subscription must not read COMPLETE. + */ + $sStatus = $oImport->error_count || $oImport->warning_count + ? Status::PARTIAL->value + : Status::COMPLETE->value; + + if (!$oModel->update($oImport->id, [ + 'log_id' => $aLog['log_id'], + 'finished' => $oNow->format('Y-m-d H:i:s'), + 'error' => $sError, + 'status' => $sStatus, + ])) { + $this->report($oImport, sprintf( + 'failed to record completion; %s', + $oModel->lastError() ?: 'no reason was reported' + )); + } + + $oImport = $this->refresh($oImport); + + /** + * Fired before the notification, so a hook which refuses the finished + * job has its reason in the email as well as in the modal. + */ + $oImport = $this->fireImportComplete($oImport); + + $this->report( + $oImport, + sprintf( + '%s; %s created, %s warned, %s errored, log %s', + $sStatus, + number_format($oImport->success_count), + number_format($oImport->warning_count), + number_format($oImport->error_count), + $oImport->log_id ? '#' . $oImport->log_id : 'not attached' + ), + null, + static::LOG_INFO + ); + + $this->notify( + $oImport, + Factory::factory('EmailUserImportComplete', Constants::MODULE_SLUG) + ); + + return $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * Rejects the job + * + * @throws FactoryException + * @throws ModelException + */ + protected function fail( + Resource\User\Import $oImport, + string $sError, + ?Throwable $e = null + ): Resource\User\Import { + + $oModel = $this->getModel(); + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + /** + * Warnings are logged alongside the errors: a job which warned and then + * failed would otherwise drop the accounts which need attention from the + * only record which names them. + */ + $aLog = $oImport->error_count || $oImport->warning_count + ? $this->attachLog($oImport, [ItemStatus::ERROR, ItemStatus::WARNING]) + : ['log_id' => $oImport->log_id, 'error' => null]; + + if ($aLog['error']) { + $sError = $this->appendLogFailure($sError, $aLog['error']); + } + + $sError = $this->truncateError($sError); + + /** + * Reported rather than thrown: throwing here would re-enter process()'s + * catch and recurse straight back into fail(). Left alone, the job's + * fingerprint does not move and the runner's stall path picks it up. + */ + if (!$oModel->update($oImport->id, [ + 'log_id' => $aLog['log_id'], + 'finished' => $oNow->format('Y-m-d H:i:s'), + 'error' => $sError, + 'status' => Status::FAILED->value, + ])) { + $this->report($oImport, sprintf( + 'failed to record the failure; %s', + $oModel->lastError() ?: 'no reason was reported' + )); + } + + $this->report($oImport, sprintf('FAILED; %s', $this->firstLine($sError)), $e); + + $oImport = $this->refresh($oImport); + + $this->fireImportFailed($oImport); + + $this->notify( + $oImport, + Factory::factory('EmailUserImportFailed', Constants::MODULE_SLUG) + ); + + return $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * Emails the person who requested the import + * + * Sent rather than queued: this is the operator's feedback on the job, and + * queueing it would put it behind the welcome email of every row the import + * created. One inline send at the end of a finished job is a price worth + * paying for them finding out promptly whether it worked. + * + * Counts, links and a redacted error only. The failing rows quote cell + * values straight out of the CSV - an email address, a name - and an inbox + * is the wrong place to keep those. Whoever needs to know which rows failed + * opens the log, which is behind a permission check. + */ + protected function notify(Resource\User\Import $oImport, Complete|Failed $oEmail): void + { + $iRecipient = $oImport->created_by instanceof \Nails\Common\Resource + ? $oImport->created_by->id + : $oImport->created_by; + + if (empty($iRecipient)) { + return; + } + + try { + + $oEmail + ->to($iRecipient) + ->data([ + 'import' => [ + 'row_count' => $oImport->row_count, + 'success' => $oImport->success_count, + 'warnings' => $oImport->warning_count, + 'errors' => $oImport->error_count, + 'url' => siteUrl('admin/auth/import/preview/' . $oImport->id), + ], + 'error' => $this->summariseError($oImport->error), + /** + * The guarded admin route rather than the CDN object: the + * log carries personal data, so the URL which reaches it is + * minted behind a permission check and expires. + */ + 'log_url' => $oImport->log_id + ? siteUrl('admin/auth/import/log/' . $oImport->id) + : null, + ]) + ->send(); + + } catch (Throwable $e) { + /** + * An email which cannot be sent must not undo the import. The + * exception is deliberately not handed to the error handler: a bad + * mail configuration would otherwise raise an alert on every import. + */ + $this->report( + $oImport, + sprintf('could not send the notification email; %s', $e->getMessage()), + null, + static::LOG_WARNING + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Generates the log CSV, puts it in the CDN, and returns its object ID + * + * The ID is returned rather than a resource deliberately. Cdn::objectCreate() + * hands back a plain stdClass - Cdn::getObject() is declared `bool|stdClass` + * and never returns a Cdn\Resource\CdnObject - so declaring a resource + * return type here raises a TypeError on the way out, which the caller then + * has to know to catch. An int cannot be got wrong, and both callers only + * ever read the ID. The admin upload path converts via the Object model + * instead; see Admin\Import::uploadCsv(). + * + * @param iterable $aItems + * + * @return int The CDN object ID of the stored log + * @throws LogException If the log cannot be built or stored + */ + protected function uploadLog(Resource\User\Import $oImport, iterable $aItems): int + { + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + $oCdn = $this->getCdn(); + + try { + $sLogPath = $oCsv->writeLog($this->getSourcePath($oImport), $aItems); + + } catch (Throwable $e) { + throw new LogException( + sprintf('Failed to build the import log; %s', $e->getMessage()), + 0, + $e + ); + } + + try { + + $mLog = $oCdn->objectCreate( + $sLogPath, + static::IMPORT_BUCKET, + [ + 'no-md5-check' => true, + 'Content-Type' => 'text/csv', + 'filename_display' => sprintf( + 'user-import-log-%s.csv', + $oNow->format('Y-m-d_H-i-s') + ), + 'metadata' => $this->getLogMetaData($oImport), + ] + ); + + } finally { + @unlink($sLogPath); + } + + /** + * objectCreate() signals failure by returning false and stashing the + * reason on the service, so it has to be read out explicitly or it is + * lost. lastError() returns false - not '' - for an empty error stack, + * hence the fallback. + */ + if (empty($mLog) || empty($mLog->id)) { + throw new LogException(sprintf( + 'Failed to store the import log in the CDN; %s', + $oCdn->lastError() ?: 'no reason was reported' + )); + } + + return (int) $mLog->id; + } + + // -------------------------------------------------------------------------- + + /** + * Attaches the log to the job, keeping hold of the reason if it cannot be + * + * The single place a log is attached. A log which cannot be written must not + * mask the reason the job finished the way it did - but it must not vanish + * either, which is what used to happen. Throwable rather than LogException + * on purpose: a programming error in here must still not take the job down, + * but it must be reported. + * + * @param ItemStatus[]|null $aStatuses The statuses to log, null for every item + * + * @return array{log_id: int|null, error: string|null} + * @throws FactoryException + * @throws ModelException + */ + protected function attachLog(Resource\User\Import $oImport, ?array $aStatuses = null): array + { + try { + return [ + 'log_id' => $this->uploadLog($oImport, $this->streamItems($oImport, $aStatuses)), + 'error' => null, + ]; + + } catch (Throwable $e) { + $this->report($oImport, 'failed to attach the log', $e); + + return [ + 'log_id' => $oImport->log_id, + 'error' => $e->getMessage(), + ]; + } + } + + // -------------------------------------------------------------------------- + + /** + * Records something noteworthy about a job + * + * Everything goes to the application log prefixed with the job's ID, so + * `grep 'import #6' application/logs/log-*.php` is the recovery path for a + * job whose stored error is not enough. A per-job log file would be tidier + * but nothing in the UI could link to it, which is the very problem this is + * meant to solve. + */ + protected function report( + Resource\User\Import $oImport, + string $sMessage, + ?Throwable $e = null, + string $sLevel = self::LOG_ERROR + ): void { + + $sLine = sprintf('User import #%s: %s', $oImport->id, $sMessage); + + if ($e !== null) { + $sLine .= ' ' . json_encode($this->errorPayload($e), JSON_UNESCAPED_SLASHES); + } + + try { + match ($sLevel) { + static::LOG_INFO => $this->getLogger()->info($sLine), + static::LOG_WARNING => $this->getLogger()->warning($sLine), + default => $this->getLogger()->error($sLine), + }; + } catch (Throwable $eLogger) { + // A worker which cannot log must still finish the job + } + + if ($e === null || $sLevel !== static::LOG_ERROR) { + return; + } + + /** + * Hand the exception to whatever the app reports errors with - Sentry, + * Rollbar - without halting; see module-cron's Run::logException(). + */ + try { + $sDriver = Factory::service('ErrorHandler')::getDriverClass(); + $sDriver::exception($e, false); + } catch (Throwable $eHandler) { + // A reporting failure must never take the job with it + } + } + + // -------------------------------------------------------------------------- + + /** + * Reduces a Throwable to the detail worth keeping + * + * Mirrors the shape module-queue records for a failed job; see + * Nails\Queue\Service\Manager::buildErrorPayload(). Deliberately duplicated + * rather than called: that method is protected, and module-queue is only a + * dev/suggested dependency, so a hard reference would break every install + * without it. + * + * @return array{type: string, message: string, code: mixed, file: string, line: int, trace: string, occurred_at: string} + */ + protected function errorPayload(Throwable $e): array + { + $sTrace = $e->getTraceAsString(); + + if (mb_strlen($sTrace) > static::ERROR_TRACE_MAX_LENGTH) { + $sTrace = mb_substr($sTrace, 0, static::ERROR_TRACE_MAX_LENGTH) . '…'; + } + + return [ + 'type' => $e::class, + 'message' => $e->getMessage(), + 'code' => $e->getCode(), + 'file' => $e->getFile(), + 'line' => $e->getLine(), + 'trace' => $sTrace, + 'occurred_at' => (new \DateTime())->format('Y-m-d H:i:s'), + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Yields the job's items, in line order, in pages + * + * Takes a set of statuses rather than one: a failed job wants its errors and + * its warnings in the same log, and asking twice would interleave two sorted + * streams for no gain. + * + * @param ItemStatus[]|null $aStatuses The statuses to yield, null for every item + * + * @return Generator + * @throws FactoryException + * @throws ModelException + */ + protected function streamItems(Resource\User\Import $oImport, ?array $aStatuses = null): Generator + { + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $iOffset = 0; + + do { + + $oDb + ->select('line, user_id, status, message') + ->where('import_id', $oImport->id) + ->order_by('line', 'asc') + ->limit(static::LOG_PAGE_SIZE, $iOffset); + + if (!empty($aStatuses)) { + $oDb->where_in('status', array_map( + fn(ItemStatus $oStatus): string => $oStatus->value, + $aStatuses + )); + } + + $aRows = $oDb->get($oItemModel->getTableName())->result(); + + foreach ($aRows as $oRow) { + yield (int) $oRow->line => [ + 'id' => $oRow->user_id === null ? null : (int) $oRow->user_id, + 'status' => $oRow->status, + 'message' => $oRow->message, + ]; + } + + $iOffset += static::LOG_PAGE_SIZE; + + } while (count($aRows) === static::LOG_PAGE_SIZE); + } + + // -------------------------------------------------------------------------- + + /** + * Returns the lines, within the given set, which already have an item + * + * @param int[] $aLines + * + * @return int[] + * @throws FactoryException + * @throws ModelException + */ + protected function getHandledLines(Resource\User\Import $oImport, array $aLines): array + { + if (empty($aLines)) { + return []; + } + + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aRows = $oDb + ->select('line') + ->where('import_id', $oImport->id) + ->where_in('line', $aLines) + ->get($oItemModel->getTableName()) + ->result(); + + return array_map(fn($oRow) => (int) $oRow->line, $aRows); + } + + // -------------------------------------------------------------------------- + + /** + * Records the outcome of a line + * + * The unique key on (import_id, line) is what makes resuming idempotent; a + * clash simply means an earlier run already dealt with this line. + * + * @return int|null The new item's ID, or null if the line was already handled + * @throws FactoryException + */ + protected function recordItem( + Resource\User\Import $oImport, + int $iLine, + ItemStatus $oStatus, + ?string $sMessage = null, + ?int $iUserId = null + ): ?int { + + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + try { + + $iItemId = $oItemModel->create([ + 'import_id' => $oImport->id, + 'line' => $iLine, + 'user_id' => $iUserId, + 'status' => $oStatus->value, + 'message' => $sMessage, + ]); + + } catch (Throwable $e) { + /** + * A clash on the (import_id, line) unique key means an earlier run + * already dealt with this line, which is how resuming stays + * idempotent - the expected path, not a fault. Anything else means + * the row will silently never be imported, which is worth the one + * extra query to tell apart. + */ + if (empty($this->getHandledLines($oImport, [$iLine]))) { + $this->report($oImport, sprintf('line %s could not be recorded', $iLine), $e); + } + + return null; + } + + return $iItemId ?: null; + } + + // -------------------------------------------------------------------------- + + /** + * Updates a previously claimed line with what actually happened + * + * @throws FactoryException + * @throws ModelException + */ + protected function resolveItem( + int $iItemId, + ItemStatus $oStatus, + ?string $sMessage = null, + ?int $iUserId = null + ): void { + + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + + $oItemModel->update($iItemId, [ + 'status' => $oStatus->value, + 'message' => $sMessage, + 'user_id' => $iUserId, + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Clears down a previous attempt at the job + * + * A job restarted from PENDING has to be cleared first: the unique key on + * (import_id, line) would otherwise make every line look as though it had + * already been handled, and the counts would be re-derived from the previous + * run's items. + * + * @throws FactoryException + * @throws ModelException + */ + protected function clearPreviousAttempt(Resource\User\Import $oImport): void + { + $this->getModel()->deleteItems($oImport->id); + + if (empty($oImport->log_id)) { + return; + } + + /** + * begin() is about to null log_id, and auth:user:import:clean only reaps + * objects it can still reach from a job row, so the previous log is + * destroyed here or not at all. + */ + try { + if (!$this->getCdn()->objectDestroy($oImport->log_id)) { + $this->report( + $oImport, + sprintf( + 'failed to destroy the previous log #%s; %s', + $oImport->log_id, + $this->getCdn()->lastError() ?: 'no reason was reported' + ), + null, + static::LOG_WARNING + ); + } + + } catch (Throwable $e) { + $this->report( + $oImport, + sprintf('failed to destroy the previous log #%s', $oImport->log_id), + $e + ); + } + } + + // -------------------------------------------------------------------------- + + /** + * Composes the job's error + * + * Plain text with newlines. Line 1 is the only line a non-technical admin + * has to read - it is what the list cell and the preview alert show - and + * everything below it is for whoever has to fix the CSV. + * + * Deliberately not translated: this is a stored audit record written by a + * worker whose locale is the app default rather than the reader's, so it is + * written in English and translated at presentation if it ever needs to be. + * + * @param array $aSample + */ + protected function composeError( + string $sSummary, + ?string $sPhase = null, + ?string $sCause = null, + ?string $sWhere = null, + array $aSample = [], + int $iTotal = 0 + ): string { + + $aBlocks = [$sSummary]; + + $aFacts = array_filter([ + $sPhase !== null ? static::ERROR_FACT_PHASE . $sPhase : null, + $sCause !== null ? 'Cause: ' . $sCause : null, + $sWhere !== null ? 'Where: ' . $sWhere : null, + ]); + + if (!empty($aFacts)) { + $aBlocks[] = implode("\n", $aFacts); + } + + if (!empty($aSample)) { + + $aLines = [ + $iTotal > count($aSample) + ? sprintf( + 'Rows with errors (showing the first %s of %s):', + number_format(count($aSample)), + number_format($iTotal) + ) + : sprintf('Rows with errors (%s):', number_format(count($aSample))), + ]; + + foreach ($aSample as $aItem) { + $aLines[] = sprintf( + ' Line %s: %s', + $aItem['line'], + $aItem['message'] ?: '(no reason was recorded)' + ); + } + + $aBlocks[] = implode("\n", $aLines); + } + + return implode("\n\n", $aBlocks); + } + + // -------------------------------------------------------------------------- + + /** + * Composes the error for a job rejected during validation + * + * @throws FactoryException + * @throws ModelException + */ + protected function composeValidationError(Resource\User\Import $oImport): string + { + return $this->composeError( + sprintf( + '%s %s in the CSV could not be validated, so no user accounts were created. Correct them and upload the file again.', + number_format($oImport->error_count), + $oImport->error_count === 1 ? 'row' : 'rows' + ), + $this->describePhase($oImport), + null, + null, + $this->sampleErrors($oImport, static::ERROR_SAMPLE_SIZE), + $oImport->error_count + ); + } + + // -------------------------------------------------------------------------- + + /** + * Composes the error for a job brought down by something unexpected + * + * @throws FactoryException + * @throws ModelException + */ + protected function composeThrowableError(Resource\User\Import $oImport, Throwable $e): string + { + $sSummary = 'The import stopped unexpectedly and did not finish.'; + + $sSurvivors = $this->describeSurvivors($oImport); + if ($sSurvivors !== null) { + $sSummary .= ' ' . $sSurvivors; + } + + return $this->composeError( + $sSummary, + $this->describePhase($oImport), + sprintf('%s - %s', $e::class, $e->getMessage()), + /** + * basename() only - an absolute path from inside a container is + * noise to an admin, and this string reaches their inbox. + */ + sprintf('%s line %s', basename($e->getFile()), $e->getLine()), + $this->sampleErrors($oImport, static::ERROR_SAMPLE_SIZE), + $oImport->error_count + ) . "\n\nThe full technical detail was written to the application log."; + } + + // -------------------------------------------------------------------------- + + /** + * Composes the error for a job which stopped making progress + * + * @throws FactoryException + * @throws ModelException + */ + protected function composeStallError(Resource\User\Import $oImport, string $sReason): string + { + $sSummary = 'The import made no progress and was stopped to prevent it looping.'; + + $sSurvivors = $this->describeSurvivors($oImport); + if ($sSurvivors !== null) { + $sSummary .= ' ' . $sSurvivors; + } + + return $this->composeError( + $sSummary, + $this->describePhase($oImport), + $sReason, + null, + $this->sampleErrors($oImport, static::ERROR_SAMPLE_SIZE), + $oImport->error_count + ); + } + + // -------------------------------------------------------------------------- + + /** + * Describes what survived a job which stopped part way through + * + * validate()'s "no user accounts were created" cannot be reused once run() + * has started: accounts may well exist by then, and telling an admin none + * were created when some were is worse than saying nothing at all. + */ + protected function describeSurvivors(Resource\User\Import $oImport): ?string + { + /** + * Warnings count as created: the account exists, it is only the work + * which was meant to follow it which did not happen. + */ + $iCreated = $oImport->success_count + $oImport->warning_count; + + if ($iCreated) { + return sprintf( + '%s user %s had already been created and %s been kept.', + number_format($iCreated), + $iCreated === 1 ? 'account' : 'accounts', + $iCreated === 1 ? 'has' : 'have' + ); + + } elseif ($oImport->status === Status::VALIDATING) { + return 'No user accounts were created.'; + } + + return null; + } + + // -------------------------------------------------------------------------- + + /** + * Describes how far the job had got + */ + protected function describePhase(Resource\User\Import $oImport): string + { + return match ($oImport->status) { + Status::PENDING => 'opening the job', + Status::VALIDATING => sprintf( + 'validating the CSV (%s of %s rows checked)', + number_format($oImport->validated_count), + number_format((int) $oImport->row_count) + ), + Status::RUNNING => sprintf( + 'creating user accounts (%s of %s rows processed)', + number_format($oImport->processed_count), + number_format((int) $oImport->row_count) + ), + default => sprintf('finishing the job (%s)', $oImport->status->value), + }; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the job's first failing lines + * + * The log CSV carries every row, but it sits behind a download; this is what + * the modal, the CLI and the database itself show, so it is worth one + * bounded query. Deliberately not the email - see notify(). + * + * @return array + * @throws FactoryException + * @throws ModelException + */ + protected function sampleErrors(Resource\User\Import $oImport, int $iLimit): array + { + return $this->sampleItems($oImport, ItemStatus::ERROR, $iLimit); + } + + + // -------------------------------------------------------------------------- + + /** + * Returns the job's first rows of a given status + * + * @return array + * @throws FactoryException + * @throws ModelException + */ + protected function sampleItems(Resource\User\Import $oImport, ItemStatus $oStatus, int $iLimit): array + { + /** @var Model\User\Import\Item $oItemModel */ + $oItemModel = Factory::model('UserImportItem', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aRows = $oDb + ->select('line, message') + ->where('import_id', $oImport->id) + ->where('status', $oStatus->value) + ->order_by('line', 'asc') + ->limit($iLimit) + ->get($oItemModel->getTableName()) + ->result(); + + return array_map( + fn($oRow): array => [ + 'line' => (int) $oRow->line, + 'message' => $oRow->message, + ], + $aRows + ); + } + + // -------------------------------------------------------------------------- + + /** + * Reduces a composed error to the part which is safe to email + * + * Keeps the summary and the phase; drops the quoted rows, the exception and + * the file it came from. The rows name people and the cause can quote one + * too - a driver reporting a duplicate key repeats the address - so none of + * it belongs in an inbox. The stored error keeps everything; this is only + * what the notification is allowed to repeat. + */ + protected function summariseError(?string $sError): ?string + { + if (empty($sError)) { + return null; + } + + $aBlocks = explode("\n\n", $sError); + $sSummary = array_shift($aBlocks); + + $aPhase = array_filter( + explode("\n", implode("\n", $aBlocks)), + fn(string $sLine): bool => str_starts_with($sLine, static::ERROR_FACT_PHASE) + ); + + return implode("\n\n", [$sSummary, ...$aPhase]); + } + + // -------------------------------------------------------------------------- + + /** + * Notes, on the end of the job's error, that the log could not be attached + */ + protected function appendLogFailure(string $sError, string $sReason): string + { + return sprintf( + "%s\n\nThe error log could not be attached: %s", + $sError, + $sReason + ); + } + + // -------------------------------------------------------------------------- + + /** + * Bounds the job's error + * + * The column is TEXT, so this guards against a pathological CSV rather than + * against the schema. + */ + protected function truncateError(string $sError): string + { + /** + * Multibyte-aware: the composed error quotes cell values straight out of + * the CSV, and cutting one of those mid-character would leave invalid + * UTF-8 in the column. + */ + return mb_strlen($sError) > static::ERROR_MAX_LENGTH + ? mb_substr($sError, 0, static::ERROR_MAX_LENGTH) . '…' + : $sError; + } + + // -------------------------------------------------------------------------- + + /** + * The summary line of a composed error, for places with room for one line + */ + protected function firstLine(string $sError): string + { + return explode("\n", $sError, 2)[0]; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the local path of the job's source CSV + * + * @throws FactoryException + * @throws ValidationException + */ + protected function getSourcePath(Resource\User\Import $oImport): string + { + $sPath = $this->getCdn()->objectLocalPath($oImport->object_id); + + if (empty($sPath)) { + throw new ValidationException( + 'Failed to get a local path for the CSV file' + ); + } + + return $sPath; + } + + // -------------------------------------------------------------------------- + + /** + * Re-reads the job so callers always see live state + * + * @throws FactoryException + * @throws ModelException + */ + protected function refresh(Resource\User\Import $oImport): Resource\User\Import + { + $oModel = $this->getModel(); + + /** @var Resource\User\Import|null $oRefreshed */ + $oRefreshed = $oModel->getById($oImport->id); + + return $oRefreshed ?? $oImport; + } + + // -------------------------------------------------------------------------- + + /** + * The system keys which mark the log as belonging to a user import + * + * The UserImport key is what stops the CDN's unused-object monitor reaping + * the log; see Cdn\Monitor\User\ImportCsv. Broken out as its own method + * because it is the one part of storing the log which needs module-cdn's + * MetaData interfaces, and so the one part this module cannot exercise on + * its own. + * + * @return array + */ + protected function getLogMetaData(Resource\User\Import $oImport): array + { + return [ + [ + 'key' => (new SystemKey\UserImport())->get(), + 'value' => true, + ], + [ + 'key' => (new SystemKey\ImportedFrom())->get(), + 'value' => $oImport->object_id, + ], + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the logger + * + * Broken out so that what a job reports can be asserted without writing to + * the filesystem, as Model\User\Import::getDb() does for the database. + * + * @throws FactoryException + */ + protected function getLogger(): Logger + { + /** @var Logger $oLogger */ + $oLogger = Factory::service('Logger'); + return $oLogger; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the CDN service + * + * Broken out so that the log-attach contract can be exercised against each + * of objectCreate()'s three outcomes without standing up a CDN. + * + * @throws FactoryException + */ + protected function getCdn(): Cdn\Service\Cdn + { + /** @var Cdn\Service\Cdn $oCdn */ + $oCdn = Factory::service('Cdn', Cdn\Constants::MODULE_SLUG); + return $oCdn; + } + + // -------------------------------------------------------------------------- + + /** + * Hands the finished job to the app, keeping hold of the reason if it objects + * + * Deliberately unable to change the job's status: every account which was + * going to be created exists by the time this runs, so a hook which throws + * is a note against a finished import, not a failed one. The reason is + * appended to `error` rather than replacing it - complete() may already have + * recorded that the log could not be attached, and both matter. + * + * @throws FactoryException + * @throws ModelException + */ + protected function fireImportComplete(Resource\User\Import $oImport): Resource\User\Import + { + try { + $this->getImportService()->onImportComplete($oImport); + + return $oImport; + + } catch (Throwable $e) { + + $this->report($oImport, 'onImportComplete() failed', $e); + + $sError = $this->truncateError(trim(sprintf( + "%s\n\n%s", + (string) $oImport->error, + sprintf( + 'The import itself finished normally, but the application ' + . 'reported a problem afterwards: %s', + $e->getMessage() + ) + ))); + + if (!$this->getModel()->update($oImport->id, ['error' => $sError])) { + $this->report($oImport, sprintf( + 'failed to record the onImportComplete() failure; %s', + $this->getModel()->lastError() ?: 'no reason was reported' + )); + } + + return $this->refresh($oImport); + } + } + + // -------------------------------------------------------------------------- + + /** + * Tells the app the job was rejected + * + * Reported and otherwise swallowed. Nothing may escape: fail() is reached + * from process()'s catch-all, so a throw from here would recurse straight + * back into it - the same reason fail()'s own update is checked rather than + * asserted. The reason is deliberately not written to the job either; that + * column holds the composed explanation of why the import was rejected, and + * this is not it. + */ + protected function fireImportFailed(Resource\User\Import $oImport): void + { + try { + $this->getImportService()->onImportFailed($oImport); + + } catch (Throwable $e) { + $this->report($oImport, 'onImportFailed() failed', $e); + } + } + + // -------------------------------------------------------------------------- + + /** + * Returns the user model + * + * Broken out alongside getImportService(): the per-row loop is where both + * row hooks fire, and what they do to a row cannot be asserted without + * standing an account creation up otherwise. + * + * @throws FactoryException + */ + protected function getUserModel(): Model\User + { + /** @var Model\User $oUserModel */ + $oUserModel = Factory::model('User', Constants::MODULE_SLUG); + return $oUserModel; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the import service, which is where an app hooks into a job + * + * Broken out for the same reason Validator::getImportService() is: the app + * subclasses it, so what a hook does to a job has to be assertable without + * standing one up. + * + * @throws FactoryException + */ + protected function getImportService(): ImportService + { + /** @var ImportService $oImportService */ + $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); + return $oImportService; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the job model + * + * @throws FactoryException + */ + protected function getModel(): Model\User\Import + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + return $oModel; + } +} diff --git a/src/Service/User/Import/Validator.php b/src/Service/User/Import/Validator.php new file mode 100644 index 00000000..3bf5e418 --- /dev/null +++ b/src/Service/User/Import/Validator.php @@ -0,0 +1,394 @@ +getImportService(); + + $aKeys = $oImportService->getKeys(); + $aUniqueKeys = $oImportService->getUniqueKeys(); + + foreach ($this->getIdentityKeys() as $sKey) { + + if (!in_array($sKey, $aKeys, true)) { + throw new TemplateException(sprintf( + 'The user import template cannot be used: accounts are identified by "%s" ' + . '(APP_NATIVE_LOGIN_USING is "%s"), but it is not among the columns returned ' + . 'by %s::getKeys().', + $sKey, + (string) Config::get('APP_NATIVE_LOGIN_USING'), + $oImportService::class + )); + } + + if (!in_array($sKey, $aUniqueKeys, true)) { + throw new TemplateException(sprintf( + 'The user import template cannot be used: accounts are identified by "%s" ' + . '(APP_NATIVE_LOGIN_USING is "%s"), so it must be among the columns returned ' + . 'by %s::getUniqueKeys().', + $sKey, + (string) Config::get('APP_NATIVE_LOGIN_USING'), + $oImportService::class + )); + } + } + } + + // -------------------------------------------------------------------------- + + /** + * The keys an account cannot be created without + * + * Mirrors the branches of Model\User::create(); note that BOTH means both are + * needed, not either. + * + * @return string[] + */ + protected function getIdentityKeys(): array + { + return match (Config::get('APP_NATIVE_LOGIN_USING')) { + 'EMAIL' => ['email'], + 'USERNAME' => ['username'], + default => ['email', 'username'], + }; + } + + // -------------------------------------------------------------------------- + + /** + * Validates the CSV's header row + * + * @param string[] $aHeader The header row + * + * @throws ValidationException + * @throws FactoryException + */ + public function validateHeader(array $aHeader): void + { + $oImportService = $this->getImportService(); + + if (empty($aHeader)) { + throw new ValidationException( + 'Missing header row' + ); + } + + $aDiff = array_diff($aHeader, $oImportService->getKeys()); + if (!empty($aDiff)) { + throw new ValidationException(sprintf( + 'Header row contains the following invalid values: %s', + implode(', ', $aDiff) + )); + } + + /** + * The diff above only catches columns we do not recognise; a CSV may + * legitimately omit most of them, and the per-field rules only apply to + * the columns actually supplied. The identity columns are the exception: + * without them Model\User::create() cannot make an account, so a file + * which omits one used to pass here and then fail on every single row. + */ + $aMissing = array_diff($this->getIdentityKeys(), $aHeader); + if (!empty($aMissing)) { + throw new ValidationException(sprintf( + 'Header row is missing the following %s, without which an account cannot be identified: %s', + count($aMissing) === 1 ? 'column' : 'columns', + implode(', ', $aMissing) + )); + } + } + + // -------------------------------------------------------------------------- + + /** + * Validates a set of raw data rows against the import service's rules + * + * @param string[] $aHeader The CSV's header row + * @param iterable $aRows Raw rows, keyed by line number + * + * @return array The errors, keyed by line number + * @throws FactoryException + * @throws NailsException + */ + public function validateRows(array $aHeader, iterable $aRows): array + { + $oImportService = $this->getImportService(); + /** @var FormValidation $oFormValidationService */ + $oFormValidationService = Factory::service('FormValidation'); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + + /** + * Key the rules by the header's own columns; the CSV need not use every + * key, nor list them in the same order as getKeys() + */ + $aValidationRules = array_filter( + array_map( + fn($sKey) => $oImportService->getValidationRules($sKey), + array_combine($aHeader, $aHeader) + ) + ); + + /** + * One validator, reused per row; each run() replaces the previous + * result so errors never leak between rows. + */ + $oRowValidator = $oFormValidationService->buildValidator( + aRules: $aValidationRules, + aData: [] + ); + + $aErrors = []; + + foreach ($aRows as $iLine => $aRow) { + + $aLineErrors = []; + + if (count($aRow) !== count($aHeader)) { + $aLineErrors[] = sprintf( + 'Row has %d columns, the header has %d', + count($aRow), + count($aHeader) + ); + } + + try { + + $oRowValidator->run($oCsv->align($aHeader, $aRow)); + + } catch (ValidationException $e) { + + foreach ($e->getData() as $sKey => $sError) { + $aLineErrors[] = sprintf('%s: %s', $sKey, $sError); + } + + } catch (Throwable $e) { + $aLineErrors[] = $e->getMessage(); + } + + if (!empty($aLineErrors)) { + $aErrors[$iLine] = $aLineErrors; + } + } + + return $aErrors; + } + + // -------------------------------------------------------------------------- + + /** + * Detects values which are duplicated within the CSV itself + * + * Duplicate detection cannot live in the per-field rules; those only ever + * see a single value, and the validator abandons a field's remaining rules + * as soon as one of them fails. It's a whole-of-file concern, so handle it + * as one streaming pass over the file, holding only the values seen so far + * for the unique keys. + * + * @param string $sPath The path to the CSV + * + * @return string[] + * @throws FactoryException + */ + public function detectDuplicates(string $sPath): array + { + $oImportService = $this->getImportService(); + /** @var Csv $oCsv */ + $oCsv = Factory::service('UserImportCsv', Constants::MODULE_SLUG); + + $aHeader = $oCsv->getHeader($sPath); + $aErrors = []; + $aSeen = []; + $aColumn = []; + + foreach ($oImportService->getUniqueKeys() as $sKey) { + + $iColumn = array_search($sKey, $aHeader, true); + if ($iColumn === false) { + continue; + } + + $aColumn[$sKey] = $iColumn; + $aSeen[$sKey] = []; + } + + if (empty($aColumn)) { + return []; + } + + foreach ($oCsv->readRawRows($sPath) as $iLine => $aRow) { + foreach ($aColumn as $sKey => $iColumn) { + + $sValue = strtolower(trim((string) ($aRow[$iColumn] ?? ''))); + if ($sValue === '') { + continue; + } + + if (array_key_exists($sValue, $aSeen[$sKey])) { + $aErrors[] = sprintf( + 'Line %d: %s: "%s" must only appear once; it is also on line %d', + $iLine, + $sKey, + $sValue, + $aSeen[$sKey][$sValue] + ); + } else { + $aSeen[$sKey][$sValue] = $iLine; + } + } + } + + return $aErrors; + } + + // -------------------------------------------------------------------------- + + /** + * Detects values which are already registered against an existing account + * + * Batched, not per row: one streaming pass collects the unique keys' values, + * then one lookup per key answers for all of them. The closures this + * replaced ran a query - two, on a hit - for every line in the file. + * + * Takes rows rather than a path so that the upload can hand it the whole + * file as a generator while the runner hands it a chunk. + * + * Note that this is an early warning, not the authority: Model\User::create() + * performs its own uniqueness check, so a value registered after this ran is + * still refused at the point of creation. + * + * @param string[] $aHeader The CSV's header row + * @param iterable $aRows Raw rows, keyed by line number + * + * @return array The errors, keyed by line number + * @throws FactoryException + * @throws TemplateException + */ + public function detectRegistered(array $aHeader, iterable $aRows): array + { + $oImportService = $this->getImportService(); + + $aColumn = []; + $aLines = []; + + foreach ($oImportService->getUniqueKeys() as $sKey) { + + $iColumn = array_search($sKey, $aHeader, true); + if ($iColumn === false) { + continue; + } + + $aColumn[$sKey] = $iColumn; + $aLines[$sKey] = []; + } + + if (empty($aColumn)) { + return []; + } + + /** + * Only the values are held, not the rows - and keyed by value, because a + * value can legitimately recur within a chunk even though the upload's + * duplicate check would reject it across the whole file. + */ + foreach ($aRows as $iLine => $aRow) { + foreach ($aColumn as $sKey => $iColumn) { + + $sValue = strtolower(trim((string) ($aRow[$iColumn] ?? ''))); + if ($sValue === '') { + continue; + } + + $aLines[$sKey][$sValue][] = $iLine; + } + } + + $aErrors = []; + + foreach ($aLines as $sKey => $aValues) { + foreach ($oImportService->whichExist($sKey, array_keys($aValues)) as $sValue) { + foreach ($aValues[$sValue] ?? [] as $iLine) { + $aErrors[$iLine][] = sprintf('%s: "%s" is already registered', $sKey, $sValue); + } + } + } + + ksort($aErrors); + + return $aErrors; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the import service + * + * Broken out so the template and unique-key contracts can be exercised + * against an app which has amended them, without registering a service; the + * same seam Model\User\Import::getDb() provides for the database. + * + * @throws FactoryException + */ + protected function getImportService(): ImportService + { + /** @var ImportService $oImportService */ + $oImportService = Factory::service('UserImport', Constants::MODULE_SLUG); + return $oImportService; + } +} diff --git a/src/Validator/User/Identifier.php b/src/Validator/User/Identifier.php new file mode 100644 index 00000000..d765b6cd --- /dev/null +++ b/src/Validator/User/Identifier.php @@ -0,0 +1,34 @@ + Config::get('APP_NATIVE_LOGIN_USING') === 'EMAIL' + ? [FormValidation::RULE_REQUIRED, FormValidation::RULE_VALID_EMAIL] + : [FormValidation::RULE_REQUIRED], + ]; + } +} diff --git a/src/Validator/User/Identity.php b/src/Validator/User/Identity.php new file mode 100644 index 00000000..f9d47f62 --- /dev/null +++ b/src/Validator/User/Identity.php @@ -0,0 +1,129 @@ +addRules(['first_name' => [FormValidation::RULE_REQUIRED]]) + * ->run($oInput->post()); + * + * Pass a user ID to exempt that user's own email/username from the uniqueness check + * when validating an edit. Override a field with `[]` to leave it unvalidated. + * + * @package Nails + * @subpackage module-auth + * @category Validator + * @author Nails Dev Team + * @link + */ + +namespace Nails\Auth\Validator\User; + +use Nails\Auth\Constants; +use Nails\Auth\Model\User; +use Nails\Common\Exception\FactoryException; +use Nails\Common\Factory\Service\FormValidation\Validator; +use Nails\Common\Service\FormValidation; +use Nails\Config; +use Nails\Factory; + +class Identity extends Validator +{ + /** + * @param int|null $iIgnoreUserId A user whose own email/username should not count as taken + */ + public function __construct(private readonly ?int $iIgnoreUserId = null) + { + parent::__construct(); + } + + // -------------------------------------------------------------------------- + + /** + * Whether the app identifies users by email + */ + public static function usesEmail(): bool + { + return in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['EMAIL', 'BOTH'], true); + } + + /** + * Whether the app identifies users by username + */ + public static function usesUsername(): bool + { + return in_array(Config::get('APP_NATIVE_LOGIN_USING'), ['USERNAME', 'BOTH'], true); + } + + // -------------------------------------------------------------------------- + + /** + * @throws FactoryException + */ + protected function rules(): array + { + $aRules = []; + + if (static::usesEmail()) { + /** @var User\Email $oUserEmailModel */ + $oUserEmailModel = Factory::model('UserEmail', Constants::MODULE_SLUG); + + $aRules['email'] = [ + 'trim', + FormValidation::RULE_REQUIRED, + FormValidation::RULE_VALID_EMAIL, + FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 255), + $this->uniqueRule($oUserEmailModel->getTableName(), 'email', 'user_id'), + ]; + } + + if (static::usesUsername()) { + /** @var User $oUserModel */ + $oUserModel = Factory::model('User', Constants::MODULE_SLUG); + + $aRules['username'] = [ + 'trim', + FormValidation::RULE_REQUIRED, + FormValidation::rule(FormValidation::RULE_MAX_LENGTH, 150), + FormValidation::RULE_ALPHA_DASH_PERIOD, + $this->uniqueRule($oUserModel->getTableName(), 'username', 'id'), + ]; + } + + return $aRules; + } + + // -------------------------------------------------------------------------- + + protected function messages(): array + { + $sKey = match (Config::get('APP_NATIVE_LOGIN_USING')) { + 'EMAIL' => 'auth_register_email_is_unique', + 'USERNAME' => 'auth_register_username_is_unique', + default => 'auth_register_identity_is_unique', + }; + + return [ + FormValidation::RULE_IS_UNIQUE => lang($sKey, siteUrl('auth/password/forgotten')), + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Compiles an `is_unique` rule, exempting the ignored user if one was given + * + * @param string $sTable The table to check + * @param string $sColumn The column holding the value + * @param string $sUserIdColumn The column holding the user ID in that table + */ + protected function uniqueRule(string $sTable, string $sColumn, string $sUserIdColumn): string + { + return $this->iIgnoreUserId + ? FormValidation::rule(FormValidation::RULE_IS_UNIQUE, $sTable, $sColumn, $this->iIgnoreUserId, $sUserIdColumn) + : FormValidation::rule(FormValidation::RULE_IS_UNIQUE, $sTable, $sColumn); + } +} diff --git a/tests/Enum/User/Import/StatusTest.php b/tests/Enum/User/Import/StatusTest.php new file mode 100644 index 00000000..d7a5b155 --- /dev/null +++ b/tests/Enum/User/Import/StatusTest.php @@ -0,0 +1,87 @@ +isTerminal()); + self::assertTrue(Status::PARTIAL->isTerminal()); + self::assertTrue(Status::FAILED->isTerminal()); + } + + // -------------------------------------------------------------------------- + + public function test_unfinished_statuses_are_not_terminal(): void + { + self::assertFalse(Status::DRAFT->isTerminal()); + self::assertFalse(Status::PENDING->isTerminal()); + self::assertFalse(Status::VALIDATING->isTerminal()); + self::assertFalse(Status::RUNNING->isTerminal()); + } + + // -------------------------------------------------------------------------- + + /** + * A draft has not been approved, so no runner may touch it + */ + public function test_a_draft_is_not_active(): void + { + self::assertFalse(Status::DRAFT->isActive()); + self::assertTrue(Status::PENDING->isActive()); + self::assertTrue(Status::VALIDATING->isActive()); + self::assertTrue(Status::RUNNING->isActive()); + } + + // -------------------------------------------------------------------------- + + /** + * Only a job a runner holds mid-flight is protected; PENDING is approved but + * untouched, so it is deletable even though it is active + */ + public function test_only_in_flight_statuses_are_not_deletable(): void + { + self::assertFalse(Status::VALIDATING->isDeletable()); + self::assertFalse(Status::RUNNING->isDeletable()); + + self::assertTrue(Status::DRAFT->isDeletable()); + self::assertTrue(Status::PENDING->isDeletable()); + self::assertTrue(Status::COMPLETE->isDeletable()); + self::assertTrue(Status::PARTIAL->isDeletable()); + self::assertTrue(Status::FAILED->isDeletable()); + } + + // -------------------------------------------------------------------------- + + /** + * Guards against `isActive()` being mistaken for the deletion rule; it is + * not, because it includes PENDING + */ + public function test_deletability_is_not_the_inverse_of_active(): void + { + self::assertTrue(Status::PENDING->isActive()); + self::assertTrue(Status::PENDING->isDeletable()); + } + + // -------------------------------------------------------------------------- + + public function test_values_maps_onto_the_database_representation(): void + { + self::assertSame( + ['PENDING', 'VALIDATING', 'RUNNING'], + Status::values(Status::active()) + ); + + self::assertSame( + ['COMPLETE', 'PARTIAL', 'FAILED'], + Status::values(Status::terminal()) + ); + } +} diff --git a/tests/Model/User/ImportModelTest.php b/tests/Model/User/ImportModelTest.php new file mode 100644 index 00000000..65ca549e --- /dev/null +++ b/tests/Model/User/ImportModelTest.php @@ -0,0 +1,229 @@ +getProperty('CACHING_ENABLED') + ->getValue() + ); + } + + // -------------------------------------------------------------------------- + + public function test_the_expected_relationships_are_registered(): void + { + $aFields = []; + foreach ((new Import())->getExpandableFields() as $oField) { + $aFields[$oField->trigger] = $oField; + } + + self::assertArrayHasKey('user', $aFields); + self::assertSame('created_by', $aFields['user']->id_column); + self::assertSame(Constants::MODULE_SLUG, $aFields['user']->provider); + + self::assertArrayHasKey('object', $aFields); + self::assertSame('object_id', $aFields['object']->id_column); + + self::assertArrayHasKey('log', $aFields); + self::assertSame('log_id', $aFields['log']->id_column); + } + + // -------------------------------------------------------------------------- + + public function test_claiming_guards_against_a_job_which_is_already_held(): void + { + $oSpy = new DatabaseSpy(); + $oModel = new ImportModelWithSpy($oSpy); + + $oModel->claim(7, 'token-a'); + + // The update only lands if nobody else holds the job... + self::assertContains(['claim_token', null], $oSpy->callsTo('where')); + + // ...and only for a status a runner is allowed to pick up + self::assertContains( + ['status', ['PENDING', 'VALIDATING', 'RUNNING']], + $oSpy->callsTo('where_in') + ); + + self::assertSame( + Status::values(Status::active()), + ['PENDING', 'VALIDATING', 'RUNNING'] + ); + + self::assertContains(['claim_token', 'token-a'], $oSpy->callsTo('set')); + self::assertContains('update', $oSpy->methods()); + } + + // -------------------------------------------------------------------------- + + public function test_a_claim_is_confirmed_by_reading_the_row_back(): void + { + $oSpy = new DatabaseSpy(); + $oSpy->iCountAllResults = 1; + + self::assertTrue((new ImportModelWithSpy($oSpy))->claim(7, 'token-a')); + + // The read back must be scoped to this process's own token + self::assertContains(['id', 7], $oSpy->callsTo('where')); + self::assertContains(['claim_token', 'token-a'], $oSpy->callsTo('where')); + } + + // -------------------------------------------------------------------------- + + public function test_losing_the_race_yields_no_claim(): void + { + $oSpy = new DatabaseSpy(); + $oSpy->iCountAllResults = 0; + + self::assertFalse((new ImportModelWithSpy($oSpy))->claim(7, 'token-a')); + } + + // -------------------------------------------------------------------------- + + public function test_releasing_clears_the_claim(): void + { + $oSpy = new DatabaseSpy(); + + (new ImportModelWithSpy($oSpy))->release(7); + + self::assertContains(['claim_token', null], $oSpy->callsTo('set')); + self::assertContains(['claimed', null], $oSpy->callsTo('set')); + self::assertContains(['id', 7], $oSpy->callsTo('where')); + self::assertContains('update', $oSpy->methods()); + } + + // -------------------------------------------------------------------------- + + /** + * A restarted job has to lose its items: the unique key on + * (import_id, line) would otherwise make every line look already handled. + */ + public function test_deleting_a_jobs_items_targets_only_that_job(): void + { + $oSpy = new DatabaseSpy(); + + self::assertTrue((new ImportModelWithSpy($oSpy))->deleteItems(7)); + + self::assertContains(['import_id', 7], $oSpy->callsTo('where')); + self::assertContains('delete', $oSpy->methods()); + + // The items table, not the jobs table + self::assertSame( + [Import\Item::TABLE], + array_map(fn(array $aArgs): string => $aArgs[0], $oSpy->callsTo('delete')) + ); + } + + // -------------------------------------------------------------------------- + + /** + * Builds a job resource from the columns the model would supply + */ + private function makeImport(?int $iObjectId, ?int $iLogId): Resource\User\Import + { + return new Resource\User\Import((object) [ + 'id' => 7, + 'object_id' => $iObjectId, + 'log_id' => $iLogId, + 'additional' => '{}', + 'status' => 'COMPLETE', + 'runner' => 'CRON', + 'claim_token' => null, + 'claimed' => null, + 'error' => null, + 'row_count' => 10, + 'validated_count' => 10, + 'processed_count' => 10, + 'success_count' => 10, + 'error_count' => 0, + 'started' => null, + 'finished' => null, + ]); + } + + // -------------------------------------------------------------------------- + + public function test_only_the_csv_is_destroyed_when_there_is_no_log(): void + { + $oCdn = new CdnSpy(); + + $aErrors = (new ImportModelWithSpy(new DatabaseSpy(), $oCdn)) + ->destroyObjects($this->makeImport(2, null)); + + self::assertSame([2], $oCdn->aDestroyed); + self::assertSame([], $aErrors); + } + + // -------------------------------------------------------------------------- + + public function test_both_the_csv_and_the_log_are_destroyed(): void + { + $oCdn = new CdnSpy(); + + $aErrors = (new ImportModelWithSpy(new DatabaseSpy(), $oCdn)) + ->destroyObjects($this->makeImport(2, 3)); + + self::assertSame([2, 3], $oCdn->aDestroyed); + self::assertSame([], $aErrors); + } + + // -------------------------------------------------------------------------- + + /** + * objectDestroy() reports a missing object by returning false, not by + * throwing, so the return value has to be checked + */ + public function test_a_false_return_is_collected_with_its_error(): void + { + $oCdn = new CdnSpy(); + $oCdn->aFailures = [3 => 'Nothing to destroy.']; + + $aErrors = (new ImportModelWithSpy(new DatabaseSpy(), $oCdn)) + ->destroyObjects($this->makeImport(2, 3)); + + self::assertSame([3 => 'Nothing to destroy.'], $aErrors); + } + + // -------------------------------------------------------------------------- + + public function test_a_throw_is_collected_with_its_message(): void + { + $oCdn = new CdnSpy(); + $oCdn->aThrows = [2 => 'The driver fell over']; + + $aErrors = (new ImportModelWithSpy(new DatabaseSpy(), $oCdn)) + ->destroyObjects($this->makeImport(2, 3)); + + // The log is still attempted; one bad object must not strand the other + self::assertSame([2, 3], $oCdn->aDestroyed); + self::assertSame([2 => 'The driver fell over'], $aErrors); + } +} diff --git a/tests/Resource/User/ImportTest.php b/tests/Resource/User/ImportTest.php new file mode 100644 index 00000000..3e2e457b --- /dev/null +++ b/tests/Resource/User/ImportTest.php @@ -0,0 +1,134 @@ + 1, + 'object_id' => 2, + 'log_id' => null, + 'additional' => '{}', + 'status' => 'RUNNING', + 'runner' => 'CRON', + 'claim_token' => null, + 'claimed' => null, + 'error' => null, + 'row_count' => 100, + 'validated_count' => 100, + 'processed_count' => 25, + 'success_count' => 25, + 'error_count' => 0, + 'started' => null, + 'finished' => null, + ], $aOverrides)); + } + + // -------------------------------------------------------------------------- + + /** + * Both columns arrived with the feature rather than with the table, so the + * resource has to tolerate a schema which has not caught up - the `make()` + * above deliberately omits them for that reason. + */ + public function test_the_columns_added_after_the_table_tolerate_a_lagging_schema(): void + { + $oImport = $this->make(); + + self::assertFalse($oImport->skip_registered); + self::assertSame(0, $oImport->warning_count); + } + + // -------------------------------------------------------------------------- + + public function test_a_warning_count_is_read_when_it_is_there(): void + { + self::assertSame(3, $this->make(['warning_count' => '3'])->warning_count); + } + + // -------------------------------------------------------------------------- + + public function test_the_status_is_cast_to_an_enum(): void + { + self::assertSame(Status::RUNNING, $this->make()->status); + } + + // -------------------------------------------------------------------------- + + public function test_the_runner_is_cast_to_an_enum(): void + { + self::assertSame(Runner::CRON, $this->make()->runner); + self::assertNull($this->make(['runner' => null])->runner); + } + + // -------------------------------------------------------------------------- + + public function test_the_additional_fields_are_decoded(): void + { + $oResource = $this->make(['additional' => '{"source":"crm"}']); + + self::assertInstanceOf(stdClass::class, $oResource->additional); + self::assertSame('crm', $oResource->additional->source); + } + + // -------------------------------------------------------------------------- + + public function test_missing_additional_fields_decode_to_an_empty_object(): void + { + self::assertEquals(new stdClass(), $this->make(['additional' => null])->additional); + } + + // -------------------------------------------------------------------------- + + public function test_progress_is_measured_against_the_running_cursor(): void + { + self::assertSame(25, $this->make()->getPercent()); + } + + // -------------------------------------------------------------------------- + + public function test_progress_is_measured_against_the_validating_cursor(): void + { + $oResource = $this->make([ + 'status' => 'VALIDATING', + 'validated_count' => 40, + 'processed_count' => 0, + ]); + + self::assertSame(40, $oResource->getPercent()); + } + + // -------------------------------------------------------------------------- + + public function test_a_finished_job_is_always_complete(): void + { + foreach (Status::terminal() as $oStatus) { + self::assertSame( + 100, + $this->make(['status' => $oStatus->value, 'processed_count' => 3])->getPercent(), + $oStatus->value + ); + } + } + + // -------------------------------------------------------------------------- + + public function test_a_job_of_unknown_size_reports_no_progress(): void + { + self::assertSame(0, $this->make(['row_count' => null])->getPercent()); + } +} diff --git a/tests/Service/User/Import/CsvTest.php b/tests/Service/User/Import/CsvTest.php new file mode 100644 index 00000000..292f62ec --- /dev/null +++ b/tests/Service/User/Import/CsvTest.php @@ -0,0 +1,278 @@ +oCsv = $oCsv; + } + + // -------------------------------------------------------------------------- + + protected function tearDown(): void + { + foreach ($this->aPaths as $sPath) { + @unlink($sPath); + } + $this->aPaths = []; + } + + // -------------------------------------------------------------------------- + + /** + * Writes a CSV to a temporary file and returns its path + */ + private function write(string $sContents): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-user-import-test-'); + file_put_contents($sPath, $sContents); + $this->aPaths[] = $sPath; + return $sPath; + } + + // -------------------------------------------------------------------------- + + public function test_get_header_returns_the_first_record(): void + { + $sPath = $this->write("email,first_name,last_name\na@example.com,Ada,Lovelace\n"); + + self::assertSame( + ['email', 'first_name', 'last_name'], + $this->oCsv->getHeader($sPath) + ); + } + + // -------------------------------------------------------------------------- + + public function test_get_header_trims_whitespace(): void + { + $sPath = $this->write("email , first_name\na@example.com,Ada\n"); + + self::assertSame( + ['email', 'first_name'], + $this->oCsv->getHeader($sPath) + ); + } + + // -------------------------------------------------------------------------- + + public function test_get_header_of_an_empty_file_is_empty(): void + { + self::assertSame([], $this->oCsv->getHeader($this->write(''))); + } + + // -------------------------------------------------------------------------- + + public function test_count_rows_excludes_the_header(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\nc@example.com\n"); + + self::assertSame(3, $this->oCsv->countRows($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_count_rows_ignores_blank_lines(): void + { + $sPath = $this->write("email\na@example.com\n\nb@example.com\n\n"); + + self::assertSame(2, $this->oCsv->countRows($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_read_raw_rows_keys_by_line_number(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\n"); + + self::assertSame( + [ + 2 => ['a@example.com'], + 3 => ['b@example.com'], + ], + iterator_to_array($this->oCsv->readRawRows($sPath), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_read_raw_rows_honours_the_offset_and_limit(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\nc@example.com\nd@example.com\n"); + + self::assertSame( + [ + 3 => ['b@example.com'], + 4 => ['c@example.com'], + ], + iterator_to_array($this->oCsv->readRawRows($sPath, 1, 2), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_read_raw_rows_past_the_end_is_empty(): void + { + $sPath = $this->write("email\na@example.com\n"); + + self::assertSame( + [], + iterator_to_array($this->oCsv->readRawRows($sPath, 10, 5), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_read_raw_rows_respects_quoted_separators(): void + { + $sPath = $this->write("email,last_name\n\"a@example.com\",\"Lovelace, Ada\"\n"); + + self::assertSame( + [2 => ['a@example.com', 'Lovelace, Ada']], + iterator_to_array($this->oCsv->readRawRows($sPath), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_align_keys_the_row_by_the_header(): void + { + self::assertSame( + ['email' => 'a@example.com', 'first_name' => 'Ada'], + $this->oCsv->align(['email', 'first_name'], ['a@example.com', 'Ada']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_align_pads_short_rows(): void + { + self::assertSame( + ['email' => 'a@example.com', 'first_name' => ''], + $this->oCsv->align(['email', 'first_name'], ['a@example.com']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_align_truncates_long_rows(): void + { + self::assertSame( + ['email' => 'a@example.com'], + $this->oCsv->align(['email'], ['a@example.com', 'Ada']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_align_trims_values(): void + { + self::assertSame( + ['email' => 'a@example.com'], + $this->oCsv->align(['email'], [' a@example.com ']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_apply_defaults_replaces_blank_cells(): void + { + // first_name has no default, so a blank cell becomes null rather than '' + self::assertSame( + ['email' => 'a@example.com', 'first_name' => null], + $this->oCsv->applyDefaults(['email' => 'a@example.com', 'first_name' => '']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_apply_defaults_leaves_populated_cells_alone(): void + { + self::assertSame( + ['first_name' => 'Ada'], + $this->oCsv->applyDefaults(['first_name' => 'Ada']) + ); + } + + // -------------------------------------------------------------------------- + + public function test_read_rows_combines_and_defaults(): void + { + $sPath = $this->write("email,first_name\na@example.com,Ada\nb@example.com,\n"); + + self::assertSame( + [ + 2 => ['email' => 'a@example.com', 'first_name' => 'Ada'], + 3 => ['email' => 'b@example.com', 'first_name' => null], + ], + iterator_to_array($this->oCsv->readRows($sPath), true) + ); + } + + // -------------------------------------------------------------------------- + + public function test_write_log_appends_the_outcome_columns(): void + { + $sPath = $this->write("email,first_name\na@example.com,Ada\nb@example.com,Grace\n"); + + $sLog = $this->oCsv->writeLog($sPath, [ + 2 => ['id' => 10, 'status' => 'SUCCESS', 'message' => ''], + 3 => ['id' => null, 'status' => 'ERROR', 'message' => 'Nope'], + ]); + + $this->aPaths[] = $sLog; + + self::assertSame( + implode("\n", [ + 'email,first_name,id,status,message', + 'a@example.com,Ada,10,SUCCESS,', + 'b@example.com,Grace,,ERROR,Nope', + '', + ]), + file_get_contents($sLog) + ); + } + + // -------------------------------------------------------------------------- + + public function test_write_log_only_emits_the_supplied_lines(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\nc@example.com\n"); + + $sLog = $this->oCsv->writeLog($sPath, [ + 3 => ['id' => null, 'status' => 'ERROR', 'message' => 'Nope'], + ]); + + $this->aPaths[] = $sLog; + + self::assertSame( + implode("\n", [ + 'email,id,status,message', + 'b@example.com,,ERROR,Nope', + '', + ]), + file_get_contents($sLog) + ); + } +} diff --git a/tests/Service/User/Import/ProcessorTest.php b/tests/Service/User/Import/ProcessorTest.php new file mode 100644 index 00000000..8e26377b --- /dev/null +++ b/tests/Service/User/Import/ProcessorTest.php @@ -0,0 +1,1346 @@ +oModel = new ImportModelRecorder(); + $this->oCdn = new CdnSpy(); + $this->oLogger = new LoggerSpy(); + $this->oProcessor = new ProcessorWithSpies($this->oModel, $this->oCdn, $this->oLogger); + } + + // -------------------------------------------------------------------------- + + protected function tearDown(): void + { + foreach ($this->aPaths as $sPath) { + if (is_file($sPath)) { + unlink($sPath); + } + } + + $this->aPaths = []; + } + + // -------------------------------------------------------------------------- + + /** + * Builds a job from the columns the model would supply + * + * created_by is left empty on purpose: notify() returns early without one, + * which keeps the email machinery out of these tests. + */ + private function make(array $aOverrides = []): Resource\User\Import + { + return new Resource\User\Import((object) array_merge([ + 'id' => 6, + 'object_id' => 20102, + 'log_id' => null, + 'additional' => '{}', + 'status' => 'VALIDATING', + 'runner' => 'CRON', + 'claim_token' => null, + 'claimed' => null, + 'error' => null, + 'row_count' => 1362, + 'validated_count' => 1362, + 'processed_count' => 0, + 'success_count' => 0, + 'error_count' => 2, + 'started' => null, + 'finished' => null, + 'created_by' => null, + ], $aOverrides)); + } + + // -------------------------------------------------------------------------- + + /** + * Writes a CSV for the log builder to read the source rows back out of + */ + private function makeCsv(int $iRows = 3): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-processor-test-'); + $this->aPaths[] = $sPath; + + $aLines = ['email,first_name,last_name']; + for ($i = 1; $i <= $iRows; $i++) { + $aLines[] = sprintf('user%d@example.com,First%d,Last%d', $i, $i, $i); + } + + file_put_contents($sPath, implode("\n", $aLines) . "\n"); + + return $sPath; + } + + // -------------------------------------------------------------------------- + + /** + * The two rows import #6 actually failed on + */ + private function importSixSample(): array + { + return [ + ['line' => 2, 'message' => 'email: "a.ahmad6@nhs.net" is already registered'], + ['line' => 3, 'message' => 'email: "a.botros@nhs.net" is already registered'], + ]; + } + + // -------------------------------------------------------------------------- + + /** + * A row whose account exists but whose after-create work did not happen + */ + private function importSixWarningSample(): array + { + return [ + ['line' => 4, 'message' => 'The account was created, but the subscription could not be started'], + ]; + } + + // -------------------------------------------------------------------------- + + /** + * Writes a CSV which the real validator will pass + * + * The header carries every column an account can be identified by, whichever + * APP_NATIVE_LOGIN_USING is configured, plus the one other column the + * template requires; without all three, validate() rejects the file before + * it ever reaches onImportStart(). + */ + private function makeValidCsv(int $iRows = 1): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-processor-test-'); + $this->aPaths[] = $sPath; + + $aLines = ['email,username,send_email']; + for ($i = 1; $i <= $iRows; $i++) { + $aLines[] = sprintf('user%d@example.com,user_%d,0', $i, $i); + } + + file_put_contents($sPath, implode("\n", $aLines) . "\n"); + + return $sPath; + } + + // -------------------------------------------------------------------------- + // The seams + // -------------------------------------------------------------------------- + + /** + * The seams are trivial, but a mistake in one is invisible to every other + * test in here - they all override them - so they are exercised against the + * real class. + */ + public function test_the_seams_hand_back_the_real_services(): void + { + $oProcessor = new Processor(); + $oClass = new ReflectionClass($oProcessor); + + /** + * getCdn() is left out: the real CDN service wants a cache directory + * this module does not have on its own, which is the same reason CdnSpy + * replaces its constructor. + */ + $aSeams = [ + 'getModel' => ImportModel::class, + 'getLogger' => Logger::class, + ]; + + foreach ($aSeams as $sMethod => $sExpected) { + self::assertInstanceOf( + $sExpected, + $oClass->getMethod($sMethod)->invoke($oProcessor), + $sMethod . '() should return a ' . $sExpected + ); + } + } + + // -------------------------------------------------------------------------- + // Storing the log + // -------------------------------------------------------------------------- + + /** + * The regression test for import #6. + * + * Cdn::objectCreate() hands back a plain stdClass, not a + * Cdn\Resource\CdnObject; uploadLog() used to declare the resource as its + * return type, so the TypeError on the way out was swallowed by fail()'s + * empty catch and the log was silently never attached. + */ + public function test_the_shape_the_cdn_actually_returns_is_accepted(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = (object) ['id' => '20103']; + + self::assertSame( + 20103, + $this->oProcessor->exposeUploadLog($this->make(), []) + ); + } + + // -------------------------------------------------------------------------- + + public function test_the_log_is_stored_in_the_import_bucket_as_a_csv(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + + $this->oProcessor->exposeUploadLog($this->make(), []); + + self::assertCount(1, $this->oCdn->aCreated); + + $aCreated = $this->oCdn->aCreated[0]; + self::assertSame(Processor::IMPORT_BUCKET, $aCreated['bucket']); + self::assertSame('text/csv', $aCreated['options']['Content-Type']); + self::assertTrue($aCreated['options']['no-md5-check']); + self::assertStringEndsWith('.csv', $aCreated['options']['filename_display']); + } + + // -------------------------------------------------------------------------- + + public function test_a_cdn_which_cannot_store_the_log_reports_its_reason(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'The file is too large, maximum file size is 1 B'; + + $this->expectException(LogException::class); + $this->expectExceptionMessage('The file is too large, maximum file size is 1 B'); + + $this->oProcessor->exposeUploadLog($this->make(), []); + } + + // -------------------------------------------------------------------------- + + /** + * lastError() returns false, not '', for an empty error stack - so without a + * fallback a silent CDN failure would produce a message which trails off. + */ + public function test_a_cdn_failure_with_no_reason_still_says_something(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = false; + + $this->expectException(LogException::class); + $this->expectExceptionMessage('no reason was reported'); + + $this->oProcessor->exposeUploadLog($this->make(), []); + } + + // -------------------------------------------------------------------------- + + public function test_a_log_which_cannot_be_built_is_reported_as_such(): void + { + // Never written, so the log builder cannot read the source rows + $this->oProcessor->sSourcePath = '/nonexistent/user-import.csv'; + + $this->expectException(LogException::class); + $this->expectExceptionMessage('Failed to build the import log'); + + $this->oProcessor->exposeUploadLog($this->make(), []); + } + + // -------------------------------------------------------------------------- + + public function test_a_log_which_cannot_be_attached_is_kept_hold_of_rather_than_swallowed(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'Failed to create object on storage service'; + + $aLog = $this->oProcessor->exposeAttachLog($this->make(['log_id' => 999]), [ItemStatus::ERROR]); + + // The reason comes back to the caller... + self::assertStringContainsString('Failed to create object on storage service', $aLog['error']); + + // ...it is written to the application log... + self::assertCount(1, $this->oLogger->aErrors); + self::assertStringContainsString('User import #6', $this->oLogger->aErrors[0]); + self::assertStringContainsString('failed to attach the log', $this->oLogger->aErrors[0]); + + // ...and an existing log is not thrown away in the process + self::assertSame(999, $aLog['log_id']); + } + + // -------------------------------------------------------------------------- + // fail() + // -------------------------------------------------------------------------- + + public function test_a_failure_records_the_reason_the_log_and_the_status(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + + $this->oProcessor->exposeFail($this->make(), 'Something went wrong.'); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::FAILED->value, $aUpdate['status']); + self::assertSame('Something went wrong.', $aUpdate['error']); + self::assertSame(20103, $aUpdate['log_id']); + self::assertNotEmpty($aUpdate['finished']); + } + + // -------------------------------------------------------------------------- + + public function test_a_failure_notes_on_itself_when_the_log_could_not_be_attached(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'The CDN is unreachable'; + + $this->oProcessor->exposeFail($this->make(), 'Something went wrong.'); + + $sError = $this->oModel->lastUpdate()['error']; + + self::assertStringContainsString('Something went wrong.', $sError); + self::assertStringContainsString('The error log could not be attached', $sError); + self::assertStringContainsString('The CDN is unreachable', $sError); + } + + // -------------------------------------------------------------------------- + + /** + * A job with nothing wrong per-row has no error log to build, so the CDN + * should not be troubled for one. + */ + public function test_a_failure_with_no_failing_rows_does_not_build_a_log(): void + { + $this->oProcessor->exposeFail($this->make(['error_count' => 0]), 'The CSV had no header.'); + + self::assertSame([], $this->oCdn->aCreated); + self::assertNull($this->oModel->lastUpdate()['log_id']); + } + + // -------------------------------------------------------------------------- + + /** + * Throwing here would re-enter process()'s catch and recurse back into + * fail(), so a failed write is reported and left alone. + */ + public function test_a_failure_which_cannot_be_recorded_is_reported_not_thrown(): void + { + $this->oModel->bUpdateReturn = false; + $this->oModel->sUpdateError = 'Deadlock found when trying to get lock'; + + $this->oProcessor->exposeFail($this->make(['error_count' => 0]), 'Something went wrong.'); + + self::assertStringContainsString( + 'Deadlock found when trying to get lock', + implode("\n", $this->oLogger->aErrors) + ); + } + + // -------------------------------------------------------------------------- + // complete() + // -------------------------------------------------------------------------- + + /** + * complete() used to call the log upload with no try/catch at all, so a log + * failure reached process()'s catch and re-branded a perfectly good import + * as FAILED - after the accounts had been created. + */ + public function test_a_log_failure_does_not_turn_a_finished_import_into_a_failed_one(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'The CDN is unreachable'; + + $oImport = $this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1362, + 'error_count' => 0, + 'log_id' => 555, + ]); + + $this->oProcessor->exposeComplete($oImport); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::COMPLETE->value, $aUpdate['status']); + + // An existing log must not be nulled by a failed upload + self::assertSame(555, $aUpdate['log_id']); + + // ...and the reason has to be visible somewhere + self::assertStringContainsString('The error log could not be attached', $aUpdate['error']); + self::assertStringContainsString('The CDN is unreachable', $aUpdate['error']); + } + + // -------------------------------------------------------------------------- + + public function test_an_import_with_failing_rows_completes_as_partial(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + + $this->oProcessor->exposeComplete($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1360, + 'error_count' => 2, + ])); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::PARTIAL->value, $aUpdate['status']); + self::assertNull($aUpdate['error']); + self::assertSame(20103, $aUpdate['log_id']); + } + + // -------------------------------------------------------------------------- + // begin() + // -------------------------------------------------------------------------- + + /** + * The unique key on (import_id, line) would otherwise make every line of a + * restarted job look as though it had already been handled, and the counts, + * which are re-derived rather than incremented, would come from the run + * before. + */ + public function test_restarting_a_job_clears_the_previous_attempt(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + + $oImport = $this->make([ + 'status' => 'PENDING', + 'log_id' => 20103, + 'validated_count' => 1362, + 'error_count' => 2, + ]); + + $this->oProcessor->exposeBegin($oImport); + + // The previous run's items go... + self::assertSame([6], $this->oModel->aItemsDeleted); + + // ...as does the log which described them... + self::assertSame([20103], $this->oCdn->aDestroyed); + + // ...and the job is reset to the top + $aUpdate = $this->oModel->lastUpdate(); + self::assertNull($aUpdate['log_id']); + self::assertNull($aUpdate['error']); + self::assertSame(0, $aUpdate['validated_count']); + self::assertSame(0, $aUpdate['error_count']); + self::assertSame(Status::VALIDATING->value, $aUpdate['status']); + self::assertSame(3, $aUpdate['row_count']); + } + + // -------------------------------------------------------------------------- + + public function test_a_previous_log_which_cannot_be_destroyed_does_not_stop_the_restart(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->aFailures = [20103 => 'Object does not exist']; + + $this->oProcessor->exposeBegin($this->make(['status' => 'PENDING', 'log_id' => 20103])); + + self::assertSame(Status::VALIDATING->value, $this->oModel->lastUpdate()['status']); + self::assertStringContainsString( + 'Object does not exist', + implode("\n", $this->oLogger->aWarnings) + ); + } + + // -------------------------------------------------------------------------- + // Composing the error + // -------------------------------------------------------------------------- + + public function test_a_validation_failure_names_the_rows_which_failed(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + + $sError = $this->oProcessor->exposeComposeValidationError($this->make()); + + self::assertStringContainsString('2 rows in the CSV could not be validated', $sError); + self::assertStringContainsString('no user accounts were created', $sError); + self::assertStringContainsString('validating the CSV (1,362 of 1,362 rows checked)', $sError); + self::assertStringContainsString('Line 2: email: "a.ahmad6@nhs.net" is already registered', $sError); + self::assertStringContainsString('Line 3: email: "a.botros@nhs.net" is already registered', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_a_single_failing_row_is_worded_in_the_singular(): void + { + $this->oProcessor->aSample = [['line' => 2, 'message' => 'email: is already registered']]; + + self::assertStringContainsString( + '1 row in the CSV could not be validated', + $this->oProcessor->exposeComposeValidationError($this->make(['error_count' => 1])) + ); + } + + // -------------------------------------------------------------------------- + + /** + * The symptom which made import #6's error useless: rows failed, but every + * message was empty, so there was nothing to read either way. + */ + public function test_a_row_with_no_recorded_reason_is_still_listed(): void + { + $this->oProcessor->aSample = [ + ['line' => 2, 'message' => null], + ['line' => 3, 'message' => ''], + ]; + + $sError = $this->oProcessor->exposeComposeValidationError($this->make()); + + self::assertStringContainsString('Line 2: (no reason was recorded)', $sError); + self::assertStringContainsString('Line 3: (no reason was recorded)', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_only_a_sample_of_a_large_failure_is_quoted(): void + { + $aSample = []; + for ($i = 2; $i <= 200; $i++) { + $aSample[] = ['line' => $i, 'message' => 'email: is already registered']; + } + + $this->oProcessor->aSample = $aSample; + + $sError = $this->oProcessor->exposeComposeValidationError($this->make(['error_count' => 199])); + + self::assertStringContainsString( + sprintf('showing the first %s of 199', Processor::ERROR_SAMPLE_SIZE), + $sError + ); + + // One header line plus the sample, and nothing beyond it + self::assertStringContainsString('Line 11: ', $sError); + self::assertStringNotContainsString('Line 12: ', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_an_unexpected_failure_names_the_exception_and_where_it_came_from(): void + { + $e = new RuntimeException('Something broke'); + + $sError = $this->oProcessor->exposeComposeThrowableError($this->make(), $e); + + self::assertStringContainsString('The import stopped unexpectedly', $sError); + self::assertStringContainsString('Cause: RuntimeException - Something broke', $sError); + self::assertStringContainsString('Where: ProcessorTest.php line ' . $e->getLine(), $sError); + self::assertStringContainsString('written to the application log', $sError); + + // A container's absolute paths are noise in an admin's inbox + self::assertStringNotContainsString(dirname($e->getFile()), $sError); + } + + // -------------------------------------------------------------------------- + // summariseError() + // -------------------------------------------------------------------------- + + /** + * The notification is an FYI which lands in an inbox, and the quoted rows + * repeat cell values out of the CSV. The reader is told what happened and + * sent to the log, which is behind a permission check, for the rest. + */ + public function test_the_emailed_error_keeps_the_summary_and_the_phase(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + + $sSummary = $this->oProcessor->exposeSummariseError( + $this->oProcessor->exposeComposeValidationError($this->make(['error_count' => 2])) + ); + + self::assertStringContainsString('could not be validated', $sSummary); + self::assertStringContainsString('Phase: validating the CSV', $sSummary); + } + + // -------------------------------------------------------------------------- + + public function test_the_emailed_error_drops_the_rows_it_quoted(): void + { + $this->oProcessor->aSample = [ + ['line' => 2, 'message' => 'email: "ada@example.com" is already registered'], + ]; + + $sSummary = $this->oProcessor->exposeSummariseError( + $this->oProcessor->exposeComposeValidationError($this->make(['error_count' => 1])) + ); + + self::assertStringNotContainsString('ada@example.com', $sSummary); + self::assertStringNotContainsString('Rows with errors', $sSummary); + self::assertStringNotContainsString('Line 2', $sSummary); + } + + // -------------------------------------------------------------------------- + + /** + * A driver reporting a duplicate key repeats the value which collided, so + * the cause can name somebody even when no row was quoted. + */ + public function test_the_emailed_error_drops_the_cause_and_where_it_came_from(): void + { + $e = new RuntimeException('Duplicate entry \'ada@example.com\' for key \'email\''); + + $sSummary = $this->oProcessor->exposeSummariseError( + $this->oProcessor->exposeComposeThrowableError($this->make(), $e) + ); + + self::assertStringContainsString('The import stopped unexpectedly', $sSummary); + self::assertStringNotContainsString('ada@example.com', $sSummary); + self::assertStringNotContainsString('Cause:', $sSummary); + self::assertStringNotContainsString('Where:', $sSummary); + } + + // -------------------------------------------------------------------------- + + /** + * complete() leaves the error unset on the happy path, and the template + * renders the block only when there is something in it. + */ + public function test_a_job_with_no_error_emails_no_error(): void + { + self::assertNull($this->oProcessor->exposeSummariseError(null)); + self::assertNull($this->oProcessor->exposeSummariseError('')); + } + + // -------------------------------------------------------------------------- + + /** + * complete() writes a one-block error when the log could not be attached or + * the app's hook objected; there is no phase line to keep. + */ + public function test_a_single_block_error_survives_intact(): void + { + self::assertSame( + 'The error log could not be attached.', + $this->oProcessor->exposeSummariseError('The error log could not be attached.') + ); + } + + // -------------------------------------------------------------------------- + + /** + * validate()'s wording cannot be reused once run() has started: telling an + * admin no accounts were created when some were is worse than saying nothing. + */ + public function test_a_failure_after_accounts_were_created_does_not_claim_none_were(): void + { + $sError = $this->oProcessor->exposeComposeThrowableError( + $this->make([ + 'status' => 'RUNNING', + 'processed_count' => 14, + 'success_count' => 14, + 'error_count' => 0, + ]), + new RuntimeException('Something broke') + ); + + self::assertStringContainsString('14 user accounts had already been created and have been kept', $sError); + self::assertStringNotContainsString('no user accounts were created', $sError); + self::assertStringContainsString('creating user accounts (14 of 1,362 rows processed)', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_a_stalled_job_records_why_it_was_abandoned(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + + $sError = $this->oProcessor->exposeComposeStallError($this->make(), 'no progress was made'); + + self::assertStringContainsString('made no progress and was stopped', $sError); + self::assertStringContainsString('Cause: no progress was made', $sError); + } + + // -------------------------------------------------------------------------- + + public function test_the_summary_is_the_first_line(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + + $sError = $this->oProcessor->exposeComposeValidationError($this->make()); + + self::assertSame( + '2 rows in the CSV could not be validated, so no user accounts were created. Correct them and upload the file again.', + $this->oProcessor->exposeFirstLine($sError) + ); + } + + // -------------------------------------------------------------------------- + + public function test_a_pathological_error_is_bounded(): void + { + $sError = $this->oProcessor->exposeTruncateError(str_repeat('a', Processor::ERROR_MAX_LENGTH * 2)); + + self::assertSame(Processor::ERROR_MAX_LENGTH, mb_strlen(rtrim($sError, '…'))); + self::assertStringEndsWith('…', $sError); + } + + // -------------------------------------------------------------------------- + + /** + * The error quotes cell values from the CSV, so a byte-wise cut would leave + * invalid UTF-8 in the column. + */ + public function test_truncation_does_not_split_a_character(): void + { + $sError = $this->oProcessor->exposeTruncateError(str_repeat('é', Processor::ERROR_MAX_LENGTH * 2)); + + self::assertSame($sError, mb_convert_encoding($sError, 'UTF-8', 'UTF-8')); + self::assertSame(Processor::ERROR_MAX_LENGTH, mb_strlen(rtrim($sError, '…'))); + } + + // -------------------------------------------------------------------------- + + public function test_a_short_error_is_left_alone(): void + { + self::assertSame('Something went wrong.', $this->oProcessor->exposeTruncateError('Something went wrong.')); + } + + // -------------------------------------------------------------------------- + // The technical payload + // -------------------------------------------------------------------------- + + /** + * Mirrors the shape module-queue records for a failed job; see + * Nails\Queue\Service\Manager::buildErrorPayload(). + */ + public function test_the_error_payload_carries_what_is_needed_to_diagnose(): void + { + $aPayload = $this->oProcessor->exposeErrorPayload(new RuntimeException('Something broke', 7)); + + self::assertSame( + ['type', 'message', 'code', 'file', 'line', 'trace', 'occurred_at'], + array_keys($aPayload) + ); + + self::assertSame(RuntimeException::class, $aPayload['type']); + self::assertSame('Something broke', $aPayload['message']); + self::assertSame(7, $aPayload['code']); + self::assertNotEmpty($aPayload['trace']); + } + + // -------------------------------------------------------------------------- + + public function test_a_long_trace_is_truncated(): void + { + $aPayload = $this->oProcessor->exposeErrorPayload( + new RuntimeException(str_repeat('deeply nested ', 500)) + ); + + self::assertLessThanOrEqual( + Processor::ERROR_TRACE_MAX_LENGTH + 1, + mb_strlen($aPayload['trace']) + ); + } + + // -------------------------------------------------------------------------- + // ItemStatus::WARNING + // -------------------------------------------------------------------------- + + /** + * A row whose account was created but whose follow-up failed is neither a + * success nor a failure. 500 silently failed subscriptions must not pass as + * COMPLETE, which is the whole point of the status existing. + */ + public function test_a_job_which_only_warned_finishes_partial(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aWarningSample = $this->importSixWarningSample(); + + $this->oProcessor->exposeComplete($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1361, + 'warning_count' => 1, + 'error_count' => 0, + ])); + + self::assertSame(Status::PARTIAL->value, $this->oModel->lastUpdate()['status']); + } + + // -------------------------------------------------------------------------- + + public function test_a_job_which_neither_errored_nor_warned_finishes_complete(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + + $this->oProcessor->exposeComplete($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1362, + 'warning_count' => 0, + 'error_count' => 0, + ])); + + self::assertSame(Status::COMPLETE->value, $this->oModel->lastUpdate()['status']); + } + + // -------------------------------------------------------------------------- + + /** + * The account exists; only the work which was meant to follow it does not. + * Telling an admin fewer accounts were created than there are is the same + * mistake as telling them none were. + */ + public function test_a_warned_account_counts_among_those_created(): void + { + $sSurvivors = $this->oProcessor->exposeDescribeSurvivors($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 14, + 'success_count' => 12, + 'warning_count' => 2, + 'error_count' => 0, + ])); + + self::assertSame('14 user accounts had already been created and have been kept.', $sSurvivors); + } + + // -------------------------------------------------------------------------- + + public function test_a_single_warned_account_is_worded_in_the_singular(): void + { + $sSurvivors = $this->oProcessor->exposeDescribeSurvivors($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1, + 'success_count' => 0, + 'warning_count' => 1, + 'error_count' => 0, + ])); + + self::assertSame('1 user account had already been created and has been kept.', $sSurvivors); + } + + // -------------------------------------------------------------------------- + + /** + * fail() used to ask for the errors alone, so a job which warned and then + * failed dropped the accounts which need attention from the only record + * which names them. + */ + public function test_a_failed_job_logs_its_warnings_alongside_its_errors(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aSample = $this->importSixSample(); + $this->oProcessor->aWarningSample = $this->importSixWarningSample(); + + $this->oProcessor->exposeFail( + $this->make(['status' => 'RUNNING', 'success_count' => 4, 'warning_count' => 1]), + 'Something went wrong.' + ); + + self::assertSame( + [[ItemStatus::ERROR, ItemStatus::WARNING]], + $this->oProcessor->aStreamed + ); + } + + // -------------------------------------------------------------------------- + + /** + * A job with nothing to say about any of its rows still must not build a + * log; the warning count is now part of that question. + */ + public function test_a_failure_with_only_warnings_still_builds_a_log(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oProcessor->aWarningSample = $this->importSixWarningSample(); + + $this->oProcessor->exposeFail( + $this->make(['status' => 'RUNNING', 'error_count' => 0, 'warning_count' => 1]), + 'Something went wrong.' + ); + + self::assertSame(20103, $this->oModel->lastUpdate()['log_id']); + } + + // -------------------------------------------------------------------------- + + public function test_the_log_carries_warnings_when_they_are_asked_for(): void + { + $this->oProcessor->aSample = $this->importSixSample(); + $this->oProcessor->aWarningSample = $this->importSixWarningSample(); + + $aItems = $this->oProcessor->exposeStreamItems( + $this->make(), + [ItemStatus::ERROR, ItemStatus::WARNING] + ); + + self::assertSame( + [ItemStatus::ERROR->value, ItemStatus::ERROR->value, ItemStatus::WARNING->value], + array_column($aItems, 'status') + ); + + // ...and only the errors when only they are + self::assertCount( + 2, + $this->oProcessor->exposeStreamItems($this->make(), [ItemStatus::ERROR]) + ); + } + + // -------------------------------------------------------------------------- + // Lifecycle hooks + // -------------------------------------------------------------------------- + + /** + * Wires a hook onto the processor and hands back the service it is on + */ + private function withHook(string $sHook, callable $cHook): ImportServiceStub + { + $oService = new ImportServiceStub(); + $oService->aHooks[$sHook] = $cHook; + $this->oProcessor->oImportService = $oService; + + return $oService; + } + + // -------------------------------------------------------------------------- + + /** + * The one hook which is allowed to refuse the whole job. It fires while + * nothing has been created, so it is deliberately left unwrapped and reaches + * process()'s catch - which means the admin gets the composed reason, with + * the phase and the cause, rather than a bare exception message. + */ + public function test_the_app_may_refuse_the_whole_job_before_anything_is_created(): void + { + $this->oProcessor->sSourcePath = $this->makeValidCsv(); + + $oService = $this->withHook('onImportStart', function (): void { + throw new RuntimeException('the cohort has not been set up yet'); + }); + + $oImport = $this->make([ + 'status' => 'VALIDATING', + 'skip_registered' => true, + 'row_count' => 1, + 'validated_count' => 0, + 'error_count' => 0, + ]); + + // What validate() will read back after recording its progress + $this->oModel->oNext = $this->make([ + 'status' => 'VALIDATING', + 'skip_registered' => true, + 'row_count' => 1, + 'validated_count' => 1, + 'error_count' => 0, + ]); + + $this->oProcessor->process($oImport, 100); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::FAILED->value, $aUpdate['status']); + self::assertStringContainsString('The import stopped unexpectedly', $aUpdate['error']); + self::assertStringContainsString('No user accounts were created.', $aUpdate['error']); + self::assertStringContainsString('the cohort has not been set up yet', $aUpdate['error']); + + // Refused before the job was ever handed to run() + self::assertNotContains( + Status::RUNNING->value, + array_column(array_column($this->oModel->aUpdates, 'data'), 'status') + ); + + self::assertSame(['onImportStart', 'onImportFailed'], $oService->calledHooks()); + } + + // -------------------------------------------------------------------------- + + /** + * By the time complete() runs, every account it was going to create exists, + * so a hook which objects is a note against a finished import - not a + * failed one. + */ + public function test_a_hook_which_objects_to_a_finished_job_cannot_re_brand_it(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + + $this->withHook('onImportComplete', function (): void { + throw new RuntimeException('the cohort roll-up could not be rebuilt'); + }); + + $this->oProcessor->exposeComplete($this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1362, + 'warning_count' => 0, + 'error_count' => 0, + ])); + + $aStatuses = array_column(array_column($this->oModel->aUpdates, 'data'), 'status'); + self::assertSame([Status::COMPLETE->value], array_values(array_filter($aStatuses))); + + // ...and the reason is on the job, where the modal and the email read it + $aUpdate = $this->oModel->lastUpdate(); + self::assertSame(['error'], array_keys($aUpdate)); + self::assertStringContainsString('the import itself finished normally', strtolower($aUpdate['error'])); + self::assertStringContainsString('the cohort roll-up could not be rebuilt', $aUpdate['error']); + } + + // -------------------------------------------------------------------------- + + /** + * complete() may already have recorded that the log could not be attached, + * and both reasons matter, so the hook's is appended rather than substituted. + */ + public function test_a_hooks_objection_is_appended_to_a_reason_already_recorded(): void + { + $this->oProcessor->sSourcePath = $this->makeCsv(); + $this->oCdn->mObjectCreateReturn = false; + $this->oCdn->sObjectCreateError = 'The CDN is unreachable'; + + $this->withHook('onImportComplete', function (): void { + throw new RuntimeException('the cohort roll-up could not be rebuilt'); + }); + + $oImport = $this->make([ + 'status' => 'RUNNING', + 'processed_count' => 1362, + 'success_count' => 1362, + 'warning_count' => 0, + 'error_count' => 0, + ]); + + /** + * The hook is handed the refreshed job, so this is what it - and the + * append - see; without it the recorder would hand back the resource + * this call started with, whose error is still null. + */ + $this->oModel->oNext = $this->make([ + 'status' => 'COMPLETE', + 'processed_count' => 1362, + 'success_count' => 1362, + 'warning_count' => 0, + 'error_count' => 0, + 'error' => 'The error log could not be attached: The CDN is unreachable', + ]); + + $this->oProcessor->exposeComplete($oImport); + + $sError = $this->oModel->lastUpdate()['error']; + + self::assertStringContainsString('The error log could not be attached', $sError); + self::assertStringContainsString('the cohort roll-up could not be rebuilt', $sError); + } + + // -------------------------------------------------------------------------- + + /** + * fail() is reached from process()'s catch-all, so an exception escaping a + * hook here would recurse straight back into it. + */ + public function test_a_hook_which_objects_to_a_failure_does_not_escape(): void + { + $oService = $this->withHook('onImportFailed', function (): void { + throw new RuntimeException('the cohort could not be released'); + }); + + $oImport = $this->oProcessor->exposeFail( + $this->make(['error_count' => 0]), + 'The CSV had no header.' + ); + + self::assertSame(Status::FAILED->value, $this->oModel->lastUpdate()['status']); + self::assertSame(['onImportFailed'], $oService->calledHooks()); + + // Swallowed, but never silently + self::assertNotEmpty(array_filter( + $this->oLogger->aErrors, + fn(string $sLine): bool => str_contains($sLine, 'onImportFailed() failed') + )); + } + + // -------------------------------------------------------------------------- + + /** + * The hook reads the job as it now stands, not as the call which failed it + * found it; an app deciding what to unwind needs the terminal counts. + */ + public function test_the_failure_hook_is_handed_the_job_as_it_now_stands(): void + { + $oService = $this->withHook('onImportFailed', function (): void { + }); + + $this->oModel->oNext = $this->make([ + 'status' => 'FAILED', + 'error_count' => 2, + 'error' => 'The CSV had no header.', + ]); + + $this->oProcessor->exposeFail($this->make(['error_count' => 0]), 'The CSV had no header.'); + + /** @var Resource\User\Import $oGiven */ + $oGiven = $oService->firstCall('onImportFailed')[0]; + + self::assertSame(Status::FAILED, $oGiven->status); + self::assertSame('The CSV had no header.', $oGiven->error); + } + + // -------------------------------------------------------------------------- + // run()'s row hooks + // -------------------------------------------------------------------------- + + /** + * Runs a two row job, with the user model and the hooks stubbed + * + * @return array{0: ImportServiceStub, 1: UserModelStub} + */ + private function runTwoRows(array $aHooks = [], array $aOverrides = []): array + { + $oService = new ImportServiceStub(); + $oUserStub = new UserModelStub(); + + foreach ($aHooks as $sHook => $cHook) { + $oService->aHooks[$sHook] = $cHook; + } + + $this->oProcessor->sSourcePath = $this->makeValidCsv(2); + $this->oProcessor->oImportService = $oService; + $this->oProcessor->oUserModel = $oUserStub; + + $oImport = $this->make(array_merge([ + 'status' => 'RUNNING', + 'skip_registered' => false, + 'row_count' => 2, + 'validated_count' => 2, + 'processed_count' => 0, + 'success_count' => 0, + 'error_count' => 0, + ], $aOverrides)); + + /** + * What run() reads back once it has recorded the chunk; the counts have + * to say the job is finished, or it never reaches complete(). + */ + $this->oModel->oNext = $this->make(array_merge([ + 'status' => 'RUNNING', + 'skip_registered' => false, + 'row_count' => 2, + 'validated_count' => 2, + 'processed_count' => 2, + 'success_count' => 2, + 'error_count' => 0, + ], $aOverrides)); + + $this->oProcessor->exposeRun($oImport, 100); + + return [$oService, $oUserStub]; + } + + // -------------------------------------------------------------------------- + + /** + * Whatever the hook returns is what create() is given; an app which reaches + * for a column the CSV carries but create() would discard has nowhere else + * to put it. + */ + public function test_the_data_hook_decides_what_reaches_create(): void + { + [$oService, $oUserStub] = $this->runTwoRows([ + 'prepareUserData' => function (array $aUserData, $oImport, array $aRow): array { + $aUserData['first_name'] = strtoupper((string) $aRow['username']); + return $aUserData; + }, + ]); + + self::assertSame( + ['USER_1', 'USER_2'], + array_column(array_column($oUserStub->aCreated, 'data'), 'first_name') + ); + + // ...and it saw the row as the CSV had it + self::assertSame('user_1', $oService->firstCall('prepareUserData')[2]['username']); + } + + // -------------------------------------------------------------------------- + + /** + * The additional fields are applied first and the hook runs after them, so + * an app which overrides the hook without calling parent:: still gets what + * it configured - and may overrule it. + */ + public function test_the_data_hook_runs_after_the_additional_fields(): void + { + [, $oUserStub] = $this->runTwoRows( + [ + 'prepareUserData' => function (array $aUserData): array { + $aUserData['salutation'] = strtoupper((string) $aUserData['cohort']); + return $aUserData; + }, + ], + ['additional' => '{"cohort":"autumn"}'] + ); + + self::assertSame('autumn', $oUserStub->aCreated[0]['data']['cohort']); + self::assertSame('AUTUMN', $oUserStub->aCreated[0]['data']['salutation']); + } + + // -------------------------------------------------------------------------- + + /** + * The hook runs before anything has been written, so a row it refuses is a + * plain error - and the job carries on to the next one. + */ + public function test_a_refused_row_errors_and_leaves_no_account(): void + { + [, $oUserStub] = $this->runTwoRows([ + 'prepareUserData' => function (array $aUserData): array { + if ($aUserData['username'] === 'user_1') { + throw new RuntimeException('no cohort could be resolved for this row'); + } + return $aUserData; + }, + ]); + + $aItems = $this->oProcessor->itemsByLine(); + + self::assertSame(ItemStatus::ERROR, $aItems[2]['status']); + self::assertSame('no cohort could be resolved for this row', $aItems[2]['message']); + self::assertNull($aItems[2]['user_id']); + + // No account was attempted for it, and the next row was + self::assertCount(1, $oUserStub->aCreated); + self::assertSame(ItemStatus::SUCCESS, $aItems[3]['status']); + } + + // -------------------------------------------------------------------------- + + /** + * The heart of ItemStatus::WARNING. create() commits before it returns, so + * the account cannot be taken back; the row records that it exists, names + * it, and says what did not happen. + */ + public function test_a_created_account_whose_follow_up_failed_warns_against_its_id(): void + { + [, $oUserStub] = $this->runTwoRows([ + 'afterUserCreate' => function (Resource\User $oUser): void { + if ($oUser->email === 'user1@example.com') { + throw new RuntimeException('the subscription could not be started'); + } + }, + ]); + + $aItems = $this->oProcessor->itemsByLine(); + + self::assertSame(ItemStatus::WARNING, $aItems[2]['status']); + self::assertSame( + 'The account was created, but the subscription could not be started', + $aItems[2]['message'] + ); + + // The user_id is the point: it is the thread back to the account to fix + self::assertSame(100, $aItems[2]['user_id']); + + // The account was still created, and the rest of the job ran + self::assertCount(2, $oUserStub->aCreated); + self::assertSame(ItemStatus::SUCCESS, $aItems[3]['status']); + self::assertSame(101, $aItems[3]['user_id']); + + // ...and it was reported, not swallowed + self::assertNotEmpty(array_filter( + $this->oLogger->aErrors, + fn(string $sLine): bool => str_contains($sLine, 'afterUserCreate() failed') + )); + } + + // -------------------------------------------------------------------------- + + /** + * A warned job has finished; it is only PARTIAL rather than COMPLETE. + */ + public function test_a_job_which_warned_still_reaches_complete(): void + { + $this->runTwoRows( + [ + 'afterUserCreate' => function (): void { + throw new RuntimeException('the subscription could not be started'); + }, + ], + ['success_count' => 1, 'warning_count' => 1] + ); + + $aUpdate = $this->oModel->lastUpdate(); + + self::assertSame(Status::PARTIAL->value, $aUpdate['status']); + self::assertNotNull($aUpdate['finished']); + } + + // -------------------------------------------------------------------------- + + /** + * The row is claimed as an error before anything irreversible happens, so a + * create() which refuses a row leaves the reason it gave, not the claim's + * placeholder. + */ + public function test_a_row_create_refuses_keeps_the_reason_create_gave(): void + { + $oService = new ImportServiceStub(); + $oUserStub = new UserModelStub(); + + $oUserStub->sFail = 'false'; + $oUserStub->sCreateError = 'This email is already in use.'; + + $this->oProcessor->sSourcePath = $this->makeValidCsv(1); + $this->oProcessor->oImportService = $oService; + $this->oProcessor->oUserModel = $oUserStub; + + $oImport = $this->make([ + 'status' => 'RUNNING', + 'row_count' => 1, + 'processed_count' => 0, + 'error_count' => 0, + ]); + + $this->oModel->oNext = $oImport; + + $this->oProcessor->exposeRun($oImport, 100); + + $aItems = $this->oProcessor->itemsByLine(); + + self::assertSame(ItemStatus::ERROR, $aItems[2]['status']); + self::assertSame('This email is already in use.', $aItems[2]['message']); + self::assertNull($aItems[2]['user_id']); + + // Nothing was created, so there is nothing for the after-create hook to hear about + self::assertSame(['prepareUserData'], $oService->calledHooks()); + } +} diff --git a/tests/Service/User/Import/ValidatorTest.php b/tests/Service/User/Import/ValidatorTest.php new file mode 100644 index 00000000..46744797 --- /dev/null +++ b/tests/Service/User/Import/ValidatorTest.php @@ -0,0 +1,580 @@ +oValidator = $oValidator; + + $this->mLoginUsing = Config::get('APP_NATIVE_LOGIN_USING'); + } + + // -------------------------------------------------------------------------- + + protected function tearDown(): void + { + foreach ($this->aPaths as $sPath) { + @unlink($sPath); + } + $this->aPaths = []; + + Config::set('APP_NATIVE_LOGIN_USING', $this->mLoginUsing); + } + + // -------------------------------------------------------------------------- + + private function write(string $sContents): string + { + $sPath = tempnam(sys_get_temp_dir(), 'nails-user-import-test-'); + file_put_contents($sPath, $sContents); + $this->aPaths[] = $sPath; + return $sPath; + } + + // -------------------------------------------------------------------------- + + /** + * Builds a validator over a template an app has amended + * + * @param string[]|null $aKeys + * @param string[]|null $aUniqueKeys + * @param array $aExisting + */ + private function withTemplate( + ?array $aKeys = null, + ?array $aUniqueKeys = null, + array $aExisting = [] + ): array { + $oService = new ImportServiceStub($aKeys, $aUniqueKeys, $aExisting); + $oValidator = new ValidatorWithStub($oService); + + return [$oValidator, $oService]; + } + + // -------------------------------------------------------------------------- + // validateTemplate() + // -------------------------------------------------------------------------- + + public function test_the_stock_template_is_usable(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'BOTH'); + + $this->expectNotToPerformAssertions(); + $this->oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + /** + * The template is the app's to amend, so it can remove the very column an + * account is identified by. Left unguarded that is not noticed until every + * row fails at Model\User::create(). + */ + public function test_a_template_without_the_identity_key_is_refused(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + [$oValidator] = $this->withTemplate( + aKeys: ['username', 'first_name'], + aUniqueKeys: ['username'] + ); + + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('getKeys()'); + + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + public function test_a_template_which_does_not_hold_the_identity_key_unique_is_refused(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + [$oValidator] = $this->withTemplate( + aKeys: ['email', 'first_name'], + aUniqueKeys: [] + ); + + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('getUniqueKeys()'); + + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + /** + * Removing username is legitimate when nobody logs in with it; the guard + * must not forbid customisation, only breakage. + */ + public function test_a_template_may_drop_a_key_the_login_mode_does_not_need(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + [$oValidator] = $this->withTemplate( + aKeys: ['email', 'first_name'], + aUniqueKeys: ['email'] + ); + + $this->expectNotToPerformAssertions(); + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + + public function test_both_identity_keys_are_required_of_the_template_when_logging_in_with_either(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'BOTH'); + + [$oValidator] = $this->withTemplate( + aKeys: ['email', 'first_name'], + aUniqueKeys: ['email'] + ); + + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('username'); + + $oValidator->validateTemplate(); + } + + // -------------------------------------------------------------------------- + // detectRegistered() + // -------------------------------------------------------------------------- + + public function test_registered_values_are_reported_against_their_line(): void + { + [$oValidator] = $this->withTemplate( + aExisting: ['email' => ['a.ahmad6@nhs.net', 'a.botros@nhs.net']] + ); + + self::assertSame( + [ + 2 => ['email: "a.ahmad6@nhs.net" is already registered'], + 3 => ['email: "a.botros@nhs.net" is already registered'], + ], + $oValidator->detectRegistered( + ['email', 'first_name'], + [ + 2 => ['a.ahmad6@nhs.net', 'Ahmad'], + 3 => ['a.botros@nhs.net', 'Botros'], + 4 => ['new@example.com', 'New'], + ] + ) + ); + } + + // -------------------------------------------------------------------------- + + public function test_registration_is_matched_regardless_of_case_or_padding(): void + { + [$oValidator] = $this->withTemplate( + aExisting: ['email' => ['ada@example.com']] + ); + + self::assertArrayHasKey( + 2, + $oValidator->detectRegistered(['email'], [2 => [' ADA@Example.com ']]) + ); + } + + // -------------------------------------------------------------------------- + + public function test_a_blank_value_is_not_looked_up(): void + { + [$oValidator, $oService] = $this->withTemplate( + aExisting: ['email' => ['ada@example.com']] + ); + + self::assertSame([], $oValidator->detectRegistered(['email'], [2 => [''], 3 => [' ']])); + self::assertSame([[]], array_column($oService->aLookups, 'values')); + } + + // -------------------------------------------------------------------------- + + /** + * The whole point of the pass: one lookup per key, not one per row. The + * closures this replaced cost a query - two, on a hit - every line. + */ + public function test_one_lookup_is_made_per_key_however_many_rows(): void + { + [$oValidator, $oService] = $this->withTemplate( + aUniqueKeys: ['email', 'username'], + aExisting: ['email' => ['ada@example.com']] + ); + + $aRows = []; + for ($i = 2; $i <= 500; $i++) { + $aRows[$i] = [sprintf('user%d@example.com', $i), sprintf('user%d', $i)]; + } + + $oValidator->detectRegistered(['email', 'username'], $aRows); + + self::assertSame(['email', 'username'], array_column($oService->aLookups, 'key')); + } + + // -------------------------------------------------------------------------- + + public function test_nothing_is_looked_up_when_no_unique_column_is_present(): void + { + [$oValidator, $oService] = $this->withTemplate(); + + self::assertSame([], $oValidator->detectRegistered(['first_name'], [2 => ['Ada']])); + self::assertSame([], $oService->aLookups); + } + + // -------------------------------------------------------------------------- + + /** + * A value may recur inside a chunk even though the upload's duplicate check + * would reject it across the whole file, and both lines must be reported. + */ + public function test_a_value_repeated_within_the_rows_is_reported_on_every_line(): void + { + [$oValidator] = $this->withTemplate( + aExisting: ['email' => ['ada@example.com']] + ); + + self::assertSame( + [2, 7], + array_keys($oValidator->detectRegistered( + ['email'], + [2 => ['ada@example.com'], 5 => ['new@example.com'], 7 => ['ada@example.com']] + )) + ); + } + + // -------------------------------------------------------------------------- + + /** + * A uniqueness check which quietly answers "none of them" is worse than one + * which fails, so an app adding a unique key must say where it lives. + */ + public function test_a_unique_key_which_cannot_be_looked_up_is_refused(): void + { + $oService = new ImportServiceStub(null, ['staff_number']); + $oService->aUnsupported = ['staff_number']; + $oValidator = new ValidatorWithStub($oService); + + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('staff_number'); + + $oValidator->detectRegistered(['staff_number'], [2 => ['12345']]); + } + + // -------------------------------------------------------------------------- + + public function test_a_valid_header_is_accepted(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + $this->expectNotToPerformAssertions(); + $this->oValidator->validateHeader(['email', 'first_name']); + } + + // -------------------------------------------------------------------------- + + public function test_the_header_need_not_use_every_key(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + $this->expectNotToPerformAssertions(); + $this->oValidator->validateHeader(['email']); + } + + // -------------------------------------------------------------------------- + + /** + * A header may omit almost anything, but not the column an account is + * identified by - Model\User::create() cannot make an account without it, so + * such a file used to fail once per row instead of once. + */ + public function test_a_header_missing_the_identity_column_is_rejected(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('email'); + + $this->oValidator->validateHeader(['first_name', 'last_name']); + } + + // -------------------------------------------------------------------------- + + public function test_the_identity_column_follows_the_login_mode(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'USERNAME'); + + // Username alone is enough... + $this->oValidator->validateHeader(['username']); + + // ...and email alone is not + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('username'); + + $this->oValidator->validateHeader(['email']); + } + + // -------------------------------------------------------------------------- + + /** + * BOTH means both, matching the else branch of Model\User::create(); an + * unset config falls here too. + */ + public function test_both_identity_columns_are_required_when_logging_in_with_either(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'BOTH'); + + $this->oValidator->validateHeader(['email', 'username']); + + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('username'); + + $this->oValidator->validateHeader(['email']); + } + + // -------------------------------------------------------------------------- + + public function test_a_missing_header_is_rejected(): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Missing header row'); + + $this->oValidator->validateHeader([]); + } + + // -------------------------------------------------------------------------- + + public function test_an_unknown_column_is_rejected(): void + { + $this->expectException(ValidationException::class); + $this->expectExceptionMessage('Header row contains the following invalid values: shoe_size'); + + $this->oValidator->validateHeader(['email', 'shoe_size']); + } + + // -------------------------------------------------------------------------- + + public function test_duplicates_are_detected(): void + { + $sPath = $this->write(implode("\n", [ + 'email,first_name', + 'a@example.com,Ada', + 'b@example.com,Grace', + 'a@example.com,Anita', + '', + ])); + + self::assertSame( + ['Line 4: email: "a@example.com" must only appear once; it is also on line 2'], + $this->oValidator->detectDuplicates($sPath) + ); + } + + // -------------------------------------------------------------------------- + + public function test_duplicate_detection_is_case_insensitive(): void + { + $sPath = $this->write("email\na@example.com\nA@EXAMPLE.COM\n"); + + self::assertCount(1, $this->oValidator->detectDuplicates($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_duplicate_detection_covers_every_unique_key(): void + { + $sPath = $this->write(implode("\n", [ + 'email,username', + 'a@example.com,ada', + 'b@example.com,ada', + 'b@example.com,grace', + '', + ])); + + self::assertSame( + [ + // Reported in line order, not grouped by key + 'Line 3: username: "ada" must only appear once; it is also on line 2', + 'Line 4: email: "b@example.com" must only appear once; it is also on line 3', + ], + $this->oValidator->detectDuplicates($sPath) + ); + } + + // -------------------------------------------------------------------------- + + public function test_blank_values_are_not_duplicates(): void + { + $sPath = $this->write("email,username\na@example.com,\nb@example.com,\n"); + + self::assertSame([], $this->oValidator->detectDuplicates($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_a_file_without_unique_columns_has_no_duplicates(): void + { + $sPath = $this->write("first_name\nAda\nAda\n"); + + self::assertSame([], $this->oValidator->detectDuplicates($sPath)); + } + + // -------------------------------------------------------------------------- + + public function test_a_clean_file_has_no_duplicates(): void + { + $sPath = $this->write("email\na@example.com\nb@example.com\n"); + + self::assertSame([], $this->oValidator->detectDuplicates($sPath)); + } + // -------------------------------------------------------------------------- + + public function test_form_validation_is_available_without_codeigniter(): void + { + self::assertFalse(function_exists('get_instance')); + self::assertInstanceOf(FormValidation::class, Factory::service('FormValidation')); + } + + // -------------------------------------------------------------------------- + + public function test_clean_rows_have_no_errors(): void + { + $aHeader = ['first_name', 'last_name', 'send_email', 'salutation', 'gender', 'dob']; + $aRows = [ + 2 => ['Ada', 'Lovelace', '1', 'Ms', '', '1815-12-10'], + 3 => ['Grace', 'Hopper', '0', '', '', ''], + ]; + + self::assertSame([], $this->oValidator->validateRows($aHeader, $aRows)); + } + + // -------------------------------------------------------------------------- + + public function test_each_error_class_is_reported_with_its_key_and_message(): void + { + $aHeader = ['send_email', 'salutation', 'last_name', 'dob']; + $aRows = [ + 2 => ['maybe', str_repeat('x', 16), str_repeat('y', 151), '2020-13-45'], + ]; + + self::assertSame( + [ + 2 => [ + 'send_email: This field must be a boolean', + 'salutation: This field is too long, maximum length is 15 characters.', + 'last_name: This field is too long, maximum length is 150 characters.', + 'dob: This field must be a valid date.', + ], + ], + $this->oValidator->validateRows($aHeader, $aRows) + ); + } + + // -------------------------------------------------------------------------- + + public function test_required_fields_fail_when_blank_but_optional_fields_do_not(): void + { + $aHeader = ['send_email', 'dob', 'salutation']; + $aRows = [2 => ['', '', '']]; + + self::assertSame( + [2 => ['send_email: This field is required.']], + $this->oValidator->validateRows($aHeader, $aRows) + ); + } + + // -------------------------------------------------------------------------- + + public function test_only_the_first_failing_rule_per_field_is_reported(): void + { + // send_email is required|is_bool; blank fails required first + $aHeader = ['send_email']; + $aRows = [2 => ['']]; + + self::assertSame( + [2 => ['send_email: This field is required.']], + $this->oValidator->validateRows($aHeader, $aRows) + ); + } + + // -------------------------------------------------------------------------- + + public function test_a_column_count_mismatch_is_reported_alongside_field_errors(): void + { + $aHeader = ['send_email', 'dob']; + $aRows = [2 => ['x']]; + + $aErrors = $this->oValidator->validateRows($aHeader, $aRows); + + self::assertSame(['Row has 1 columns, the header has 2', 'send_email: This field must be a boolean'], $aErrors[2]); + } + + // -------------------------------------------------------------------------- + + public function test_errors_are_keyed_by_line_number(): void + { + $aHeader = ['send_email']; + $aRows = [7 => ['1'], 9 => ['nope'], 12 => ['0']]; + + self::assertSame([9], array_keys($this->oValidator->validateRows($aHeader, $aRows))); + } + + // -------------------------------------------------------------------------- + + public function test_errors_do_not_leak_between_rows(): void + { + $aHeader = ['send_email']; + $aRows = [2 => ['nope'], 3 => ['1']]; + + self::assertSame([2], array_keys($this->oValidator->validateRows($aHeader, $aRows))); + } + + // -------------------------------------------------------------------------- + + public function test_the_gender_rule_uses_the_models_genders(): void + { + $aHeader = ['gender']; + $aRows = [2 => ['not-a-gender']]; + + $aErrors = $this->oValidator->validateRows($aHeader, $aRows); + + self::assertCount(1, $aErrors[2]); + self::assertStringStartsWith('gender: This field must be one of: ', $aErrors[2][0]); + } +} diff --git a/tests/Service/User/ImportTest.php b/tests/Service/User/ImportTest.php new file mode 100644 index 00000000..1b19578a --- /dev/null +++ b/tests/Service/User/ImportTest.php @@ -0,0 +1,193 @@ +oService = $oService; + + $this->mLoginUsing = Config::get('APP_NATIVE_LOGIN_USING'); + } + + // -------------------------------------------------------------------------- + + protected function tearDown(): void + { + Config::set('APP_NATIVE_LOGIN_USING', $this->mLoginUsing); + } + + // -------------------------------------------------------------------------- + + /** + * Counts the closures in a rule set; each one is a potential query per row + * + * @param array $aRules + */ + private function countClosures(array $aRules): int + { + return count(array_filter($aRules, fn($mRule): bool => $mRule instanceof Closure)); + } + + // -------------------------------------------------------------------------- + + /** + * Uniqueness used to be a closure calling getByEmail() for every row - a + * query each, two on a hit. It is now batched; see whichExist(). + */ + public function test_the_email_rules_no_longer_query_per_row(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + $aRules = $this->oService->getValidationRules('email'); + + self::assertSame(0, $this->countClosures($aRules)); + self::assertSame( + [FormValidation::RULE_REQUIRED, FormValidation::RULE_VALID_EMAIL], + array_values($aRules) + ); + } + + // -------------------------------------------------------------------------- + + /** + * The format check remains, and now actually runs: the validator abandons a + * field's remaining rules once one fails, so the uniqueness closure used to + * short-circuit it. + */ + public function test_the_username_rules_keep_only_the_format_check(): void + { + self::assertSame(1, $this->countClosures($this->oService->getValidationRules('username'))); + } + + // -------------------------------------------------------------------------- + + public function test_the_identity_keys_are_unique_keys(): void + { + foreach (['email', 'username'] as $sKey) { + self::assertContains($sKey, $this->oService->getKeys(), $sKey); + self::assertContains($sKey, $this->oService->getUniqueKeys(), $sKey); + } + } + + // -------------------------------------------------------------------------- + + public function test_nothing_is_looked_up_for_an_empty_value_list(): void + { + self::assertSame([], $this->oService->whichExist('email', [])); + self::assertSame([], $this->oService->whichExist('username', [' ', ''])); + } + + // -------------------------------------------------------------------------- + + /** + * An app which adds a unique key must say where the value lives; answering + * "none of them" would let a duplicate through while appearing to check. + * Refused even with nothing to look up, so the misconfiguration cannot hide + * behind an empty file. + */ + public function test_an_unsupported_unique_key_is_refused(): void + { + $this->expectException(TemplateException::class); + $this->expectExceptionMessage('staff_number'); + + $this->oService->whichExist('staff_number', []); + } + + // -------------------------------------------------------------------------- + // Lifecycle hooks + // -------------------------------------------------------------------------- + + /** + * Builds a job to hand the hooks + */ + private function makeImport(): Resource\User\Import + { + return new Resource\User\Import((object) [ + 'id' => 6, + 'object_id' => 2, + 'log_id' => null, + 'additional' => '{"cohort":"2026"}', + 'status' => 'RUNNING', + 'runner' => 'CRON', + 'claim_token' => null, + 'claimed' => null, + 'error' => null, + 'row_count' => 3, + 'validated_count' => 3, + 'processed_count' => 0, + 'success_count' => 0, + 'error_count' => 0, + 'started' => null, + 'finished' => null, + ]); + } + + // -------------------------------------------------------------------------- + + /** + * Every hook ships as a no-op, so an app which overrides one of them changes + * nothing else about how a job runs. + */ + public function test_the_lifecycle_hooks_do_nothing_by_default(): void + { + $oImport = $this->makeImport(); + $oUser = new Resource\User((object) ['id' => 42, 'email' => 'ada@example.com']); + + $aBefore = (array) $oImport; + + // None of them throws, and none of them touches the job it was given + $this->oService->onImportStart($oImport); + $this->oService->afterUserCreate($oUser, ['email' => 'ada@example.com'], $oImport); + $this->oService->onImportComplete($oImport); + $this->oService->onImportFailed($oImport); + + self::assertEquals($aBefore, (array) $oImport); + } + + // -------------------------------------------------------------------------- + + /** + * The guard against a future edit giving this an opinion: whatever the + * processor assembled is what reaches Model\User::create(), untouched. + */ + public function test_the_data_hook_returns_its_input_unchanged(): void + { + $aUserData = [ + 'email' => 'ada@example.com', + 'first_name' => 'Ada', + 'group_id' => 3, + ]; + + self::assertSame( + $aUserData, + $this->oService->prepareUserData( + $aUserData, + $this->makeImport(), + ['email' => 'ada@example.com', 'staff_number' => 'A1234'] + ) + ); + } +} diff --git a/tests/Stub/CdnSpy.php b/tests/Stub/CdnSpy.php new file mode 100644 index 00000000..741d1f32 --- /dev/null +++ b/tests/Stub/CdnSpy.php @@ -0,0 +1,128 @@ + + */ + public array $aDestroyed = []; + + /** + * Object IDs which objectDestroy() should report as failed by returning + * false, mapped onto the error it should then report + * + * @var array + */ + public array $aFailures = []; + + /** + * Object IDs which objectDestroy() should throw for, mapped onto the + * exception message + * + * @var array + */ + public array $aThrows = []; + + // -------------------------------------------------------------------------- + + /** + * Every set of arguments passed to objectCreate(), in order + * + * @var array + */ + public array $aCreated = []; + + /** + * What objectCreate() should hand back + * + * Defaults to a plain stdClass carrying an ID, because that is what the real + * service returns - objectCreate() ends at createObject(), which returns + * Cdn::getObject(), declared `bool|stdClass`. Set to false to model a + * failure, in which case $sObjectCreateError is reported alongside it. + */ + public mixed $mObjectCreateReturn = null; + + /** + * The error objectCreate() should report when it returns false + */ + public ?string $sObjectCreateError = null; + + /** + * The message objectCreate() should throw, if it should throw at all + */ + public ?string $sObjectCreateThrow = null; + + // -------------------------------------------------------------------------- + + public function __construct() + { + // Deliberately does not stand up a driver + } + + // -------------------------------------------------------------------------- + + public function objectCreate($object, $mBucket, $aOptions = [], $bIsStream = false) + { + $this->aCreated[] = [ + 'path' => $object, + 'bucket' => $mBucket, + 'options' => $aOptions, + ]; + + if ($this->sObjectCreateThrow !== null) { + throw new \RuntimeException($this->sObjectCreateThrow); + } + + if ($this->mObjectCreateReturn === false) { + /** + * The real service catches everything, sets an error and returns + * false; it does not throw, which is what makes the reason so easy + * to lose. + */ + if ($this->sObjectCreateError !== null) { + $this->setError($this->sObjectCreateError); + } + + return false; + } + + return $this->mObjectCreateReturn ?? (object) ['id' => 20103]; + } + + // -------------------------------------------------------------------------- + + public function objectDestroy(int|string|Resource\CdnObject|null $object): bool + { + $this->aDestroyed[] = $object; + + if (array_key_exists((int) $object, $this->aThrows)) { + throw new \RuntimeException($this->aThrows[(int) $object]); + } + + if (array_key_exists((int) $object, $this->aFailures)) { + /** + * Mirrors the real service, which reports a missing object, a driver + * failure, and a rolled back transaction by setting an error and + * returning false rather than by throwing. + */ + $this->setError($this->aFailures[(int) $object]); + return false; + } + + return true; + } +} diff --git a/tests/Stub/DatabaseSpy.php b/tests/Stub/DatabaseSpy.php new file mode 100644 index 00000000..7ef28314 --- /dev/null +++ b/tests/Stub/DatabaseSpy.php @@ -0,0 +1,80 @@ + string, 'args' => array] + * + * @var array + */ + public array $aCalls = []; + + /** + * What count_all_results() should report next + */ + public int $iCountAllResults = 0; + + // -------------------------------------------------------------------------- + + public function __construct() + { + // Deliberately does not connect + } + + // -------------------------------------------------------------------------- + + public function __call($sMethod, $aArguments) + { + $this->aCalls[] = [ + 'method' => $sMethod, + 'args' => $aArguments, + ]; + + return match ($sMethod) { + 'count_all_results' => $this->iCountAllResults, + 'update' => true, + default => $this, + }; + } + + // -------------------------------------------------------------------------- + + /** + * Returns the arguments of every call to $sMethod + */ + public function callsTo(string $sMethod): array + { + return array_values( + array_map( + fn(array $aCall) => $aCall['args'], + array_filter( + $this->aCalls, + fn(array $aCall) => $aCall['method'] === $sMethod + ) + ) + ); + } + + // -------------------------------------------------------------------------- + + /** + * The methods which were called, in order + * + * @return string[] + */ + public function methods(): array + { + return array_column($this->aCalls, 'method'); + } +} diff --git a/tests/Stub/ImportModelRecorder.php b/tests/Stub/ImportModelRecorder.php new file mode 100644 index 00000000..165663f6 --- /dev/null +++ b/tests/Stub/ImportModelRecorder.php @@ -0,0 +1,113 @@ + int, 'data' => array] + * + * @var array + */ + public array $aUpdates = []; + + /** + * What update() should report + */ + public bool $bUpdateReturn = true; + + /** + * The error update() should report when it returns false + */ + public ?string $sUpdateError = null; + + /** + * Every job ID passed to deleteItems(), in order + * + * @var int[] + */ + public array $aItemsDeleted = []; + + /** + * What getById() should hand back; null makes the processor keep the + * resource it already had + */ + public ?Resource\User\Import $oNext = null; + + // -------------------------------------------------------------------------- + + /** + * The counts the model derives are read through here, so a recorder can + * answer them without a server; see Model\User\Import::countItems() + */ + public DatabaseSpy $oDbSpy; + + // -------------------------------------------------------------------------- + + public function __construct() + { + $this->oDbSpy = new DatabaseSpy(); + parent::__construct(); + } + + // -------------------------------------------------------------------------- + + protected function getDb(): Database + { + return $this->oDbSpy; + } + + // -------------------------------------------------------------------------- + + public function update($iId, array $aData = []): bool + { + $this->aUpdates[] = [ + 'id' => (int) $iId, + 'data' => $aData, + ]; + + if (!$this->bUpdateReturn && $this->sUpdateError !== null) { + $this->setError($this->sUpdateError); + } + + return $this->bUpdateReturn; + } + + // -------------------------------------------------------------------------- + + public function getById(?int $iId, array $aData = []) + { + return $this->oNext; + } + + // -------------------------------------------------------------------------- + + public function deleteItems(int $iId): bool + { + $this->aItemsDeleted[] = $iId; + return true; + } + + // -------------------------------------------------------------------------- + + /** + * The data of the last update, or null if there has not been one + */ + public function lastUpdate(): ?array + { + $aUpdate = end($this->aUpdates); + return $aUpdate === false ? null : $aUpdate['data']; + } +} diff --git a/tests/Stub/ImportModelWithSpy.php b/tests/Stub/ImportModelWithSpy.php new file mode 100644 index 00000000..c08c218a --- /dev/null +++ b/tests/Stub/ImportModelWithSpy.php @@ -0,0 +1,34 @@ +oSpy; + } + + // -------------------------------------------------------------------------- + + protected function getCdn(): Cdn + { + return $this->oCdnSpy ?? new CdnSpy(); + } +} diff --git a/tests/Stub/ImportServiceStub.php b/tests/Stub/ImportServiceStub.php new file mode 100644 index 00000000..a81c007f --- /dev/null +++ b/tests/Stub/ImportServiceStub.php @@ -0,0 +1,193 @@ + $aExisting Values already taken, keyed by unique key + */ + public function __construct( + private ?array $aKeys = null, + private ?array $aUniqueKeys = null, + private array $aExisting = [] + ) { + } + + // -------------------------------------------------------------------------- + + /** + * Every call to whichExist(), in order: ['key' => string, 'values' => string[]] + * + * @var array + */ + public array $aLookups = []; + + /** + * Unique keys which whichExist() should refuse, modelling an app which has + * added a key without saying where it lives + * + * @var string[] + */ + public array $aUnsupported = []; + + /** + * What each lifecycle hook should do, keyed by hook name; null leaves the + * inherited no-op in place. This is the shape an app subclass takes - one + * override per hook - with the body made settable per test. + * + * @var array + */ + public array $aHooks = []; + + /** + * Every hook call, in order: ['hook' => string, 'args' => array] + * + * @var array + */ + public array $aHookCalls = []; + + // -------------------------------------------------------------------------- + + public function getKeys(): array + { + return $this->aKeys ?? parent::getKeys(); + } + + // -------------------------------------------------------------------------- + + public function getUniqueKeys(): array + { + return $this->aUniqueKeys ?? parent::getUniqueKeys(); + } + + // -------------------------------------------------------------------------- + + public function whichExist(string $sKey, array $aValues): array + { + $this->aLookups[] = [ + 'key' => $sKey, + 'values' => $aValues, + ]; + + if (in_array($sKey, $this->aUnsupported, true)) { + // Mirrors the real service's refusal to answer for a key it cannot look up + return parent::whichExist($sKey, $aValues); + } + + return array_values( + array_intersect( + array_map('strtolower', $this->aExisting[$sKey] ?? []), + array_map('strtolower', $aValues) + ) + ); + } + + // -------------------------------------------------------------------------- + // Lifecycle hooks + // -------------------------------------------------------------------------- + + public function onImportStart(Resource\User\Import $oImport): void + { + $this->fire('onImportStart', [$oImport]); + } + + // -------------------------------------------------------------------------- + + public function prepareUserData(array $aUserData, Resource\User\Import $oImport, array $aRow): array + { + $mReturn = $this->fire('prepareUserData', [$aUserData, $oImport, $aRow]); + + /** + * Falls back to the parent rather than to $aUserData: an unset hook has + * to exercise the real no-op, which is what guards against a future edit + * giving it an opinion. + */ + return $mReturn ?? parent::prepareUserData($aUserData, $oImport, $aRow); + } + + // -------------------------------------------------------------------------- + + public function afterUserCreate( + Resource\User $oUser, + array $aUserData, + Resource\User\Import $oImport + ): void { + $this->fire('afterUserCreate', [$oUser, $aUserData, $oImport]); + } + + // -------------------------------------------------------------------------- + + public function onImportComplete(Resource\User\Import $oImport): void + { + $this->fire('onImportComplete', [$oImport]); + } + + // -------------------------------------------------------------------------- + + public function onImportFailed(Resource\User\Import $oImport): void + { + $this->fire('onImportFailed', [$oImport]); + } + + // -------------------------------------------------------------------------- + + /** + * Records the call and runs whatever the test set for the hook + * + * @param array $aArgs + * + * @return mixed Whatever the hook returned, or null if none was set + */ + private function fire(string $sHook, array $aArgs): mixed + { + $this->aHookCalls[] = [ + 'hook' => $sHook, + 'args' => $aArgs, + ]; + + $cHook = $this->aHooks[$sHook] ?? null; + + return $cHook === null ? null : $cHook(...$aArgs); + } + + // -------------------------------------------------------------------------- + + /** + * The hooks which have been called, in order + * + * @return string[] + */ + public function calledHooks(): array + { + return array_column($this->aHookCalls, 'hook'); + } + + // -------------------------------------------------------------------------- + + /** + * The arguments of the first call to a hook, or null if it was not called + * + * @return array|null + */ + public function firstCall(string $sHook): ?array + { + foreach ($this->aHookCalls as $aCall) { + if ($aCall['hook'] === $sHook) { + return $aCall['args']; + } + } + + return null; + } +} diff --git a/tests/Stub/LoggerSpy.php b/tests/Stub/LoggerSpy.php new file mode 100644 index 00000000..eaa6b282 --- /dev/null +++ b/tests/Stub/LoggerSpy.php @@ -0,0 +1,68 @@ +aInfo[] = (string) $sLine; + return $this; + } + + // -------------------------------------------------------------------------- + + public function warning($sLine = ''): self + { + $this->aWarnings[] = (string) $sLine; + return $this; + } + + // -------------------------------------------------------------------------- + + public function error($sLine = ''): self + { + $this->aErrors[] = (string) $sLine; + return $this; + } + + // -------------------------------------------------------------------------- + + /** + * Every line recorded, whatever the level + * + * @return string[] + */ + public function all(): array + { + return array_merge($this->aInfo, $this->aWarnings, $this->aErrors); + } +} diff --git a/tests/Stub/ProcessorWithSpies.php b/tests/Stub/ProcessorWithSpies.php new file mode 100644 index 00000000..ffa7659e --- /dev/null +++ b/tests/Stub/ProcessorWithSpies.php @@ -0,0 +1,330 @@ + int, 'message' => string|null], ...] + * + * @var array + */ + public array $aSample = []; + + /** + * The job's warned rows, in the same shape as $aSample + * + * @var array + */ + public array $aWarningSample = []; + + /** + * The import service the hooks are fired against; the real one unless a + * test supplies its own + */ + public ?ImportService $oImportService = null; + + /** + * The user model accounts are created through; the real one unless a test + * supplies its own + */ + public ?UserModel $oUserModel = null; + + /** + * The statuses each call to streamItems() asked for, in order + * + * @var array + */ + public array $aStreamed = []; + + /** + * Every item claimed by run(), keyed by the ID it was given: + * ['line' => int, 'status' => ItemStatus, 'message' => ?string, 'user_id' => ?int] + * + * @var array + */ + public array $aItems = []; + + /** + * Lines getHandledLines() should report as already dealt with + * + * @var int[] + */ + public array $aHandled = []; + + // -------------------------------------------------------------------------- + + public function __construct( + public ImportModelRecorder $oModelStub, + public CdnSpy $oCdnSpy, + public LoggerSpy $oLoggerSpy + ) { + } + + // -------------------------------------------------------------------------- + // Seams + // -------------------------------------------------------------------------- + + protected function getModel(): ImportModel + { + return $this->oModelStub; + } + + protected function getCdn(): Cdn + { + return $this->oCdnSpy; + } + + protected function getLogger(): Logger + { + return $this->oLoggerSpy; + } + + protected function getSourcePath(Resource\User\Import $oImport): string + { + return $this->sSourcePath ?? parent::getSourcePath($oImport); + } + + protected function refresh(Resource\User\Import $oImport): Resource\User\Import + { + return $this->oModelStub->oNext ?? $oImport; + } + + /** + * The real implementation needs module-cdn's MetaData interfaces, which the + * copy this module vendors for its own tests does not carry - the same + * staleness which makes phpstan report src/Cdn/MetaData/SystemKey/*. The + * keys are not what these tests are about, so they are left out. + */ + protected function getLogMetaData(Resource\User\Import $oImport): array + { + return []; + } + + protected function getImportService(): ImportService + { + return $this->oImportService ?? parent::getImportService(); + } + + protected function getUserModel(): UserModel + { + return $this->oUserModel ?? parent::getUserModel(); + } + + /** + * The item table stands in for itself: the claim-then-resolve contract is + * what run() is built on, and asserting the outcome of a row means being + * able to read it back without a database. + */ + protected function recordItem( + Resource\User\Import $oImport, + int $iLine, + ItemStatus $oStatus, + ?string $sMessage = null, + ?int $iUserId = null + ): ?int { + + $iItemId = count($this->aItems) + 1; + + $this->aItems[$iItemId] = [ + 'line' => $iLine, + 'status' => $oStatus, + 'message' => $sMessage, + 'user_id' => $iUserId, + ]; + + return $iItemId; + } + + protected function resolveItem( + int $iItemId, + ItemStatus $oStatus, + ?string $sMessage = null, + ?int $iUserId = null + ): void { + + $this->aItems[$iItemId] = array_merge( + $this->aItems[$iItemId] ?? ['line' => 0], + [ + 'status' => $oStatus, + 'message' => $sMessage, + 'user_id' => $iUserId, + ] + ); + } + + protected function getHandledLines(Resource\User\Import $oImport, array $aLines): array + { + return array_values(array_intersect($this->aHandled, $aLines)); + } + + protected function sampleErrors(Resource\User\Import $oImport, int $iLimit): array + { + return array_slice($this->aSample, 0, $iLimit); + } + + /** + * @param ItemStatus[]|null $aStatuses + */ + protected function streamItems(Resource\User\Import $oImport, ?array $aStatuses = null): Generator + { + $this->aStreamed[] = $aStatuses; + + $aItems = array_merge( + array_map( + fn(array $aItem): array => $aItem + ['status' => ItemStatus::ERROR], + $this->aSample + ), + array_map( + fn(array $aItem): array => $aItem + ['status' => ItemStatus::WARNING], + $this->aWarningSample + ) + ); + + foreach ($aItems as $aItem) { + + if (!empty($aStatuses) && !in_array($aItem['status'], $aStatuses, true)) { + continue; + } + + yield $aItem['line'] => [ + 'id' => null, + 'status' => $aItem['status']->value, + 'message' => $aItem['message'], + ]; + } + } + + // -------------------------------------------------------------------------- + // Exposed for assertion + // -------------------------------------------------------------------------- + + public function exposeUploadLog(Resource\User\Import $oImport, iterable $aItems): int + { + return $this->uploadLog($oImport, $aItems); + } + + /** + * @param ItemStatus[]|null $aStatuses + */ + public function exposeAttachLog(Resource\User\Import $oImport, ?array $aStatuses = null): array + { + return $this->attachLog($oImport, $aStatuses); + } + + /** + * @param ItemStatus[]|null $aStatuses + * + * @return array + */ + public function exposeStreamItems(Resource\User\Import $oImport, ?array $aStatuses = null): array + { + return iterator_to_array($this->streamItems($oImport, $aStatuses), true); + } + + public function exposeDescribeSurvivors(Resource\User\Import $oImport): ?string + { + return $this->describeSurvivors($oImport); + } + + public function exposeBegin(Resource\User\Import $oImport): Resource\User\Import + { + return $this->begin($oImport); + } + + public function exposeRun(Resource\User\Import $oImport, int $iLimit): Resource\User\Import + { + return $this->run($oImport, $iLimit); + } + + /** + * The items run() recorded, in line order + * + * @return array + */ + public function itemsByLine(): array + { + $aItems = []; + + foreach ($this->aItems as $aItem) { + $aItems[$aItem['line']] = $aItem; + } + + ksort($aItems); + + return $aItems; + } + + public function exposeComplete(Resource\User\Import $oImport): Resource\User\Import + { + return $this->complete($oImport); + } + + public function exposeFail( + Resource\User\Import $oImport, + string $sError, + ?Throwable $e = null + ): Resource\User\Import { + return $this->fail($oImport, $sError, $e); + } + + public function exposeComposeValidationError(Resource\User\Import $oImport): string + { + return $this->composeValidationError($oImport); + } + + public function exposeComposeThrowableError(Resource\User\Import $oImport, Throwable $e): string + { + return $this->composeThrowableError($oImport, $e); + } + + public function exposeComposeStallError(Resource\User\Import $oImport, string $sReason): string + { + return $this->composeStallError($oImport, $sReason); + } + + public function exposeErrorPayload(Throwable $e): array + { + return $this->errorPayload($e); + } + + public function exposeTruncateError(string $sError): string + { + return $this->truncateError($sError); + } + + public function exposeFirstLine(string $sError): string + { + return $this->firstLine($sError); + } + + public function exposeSummariseError(?string $sError): ?string + { + return $this->summariseError($sError); + } +} diff --git a/tests/Stub/UserModelStub.php b/tests/Stub/UserModelStub.php new file mode 100644 index 00000000..577aede0 --- /dev/null +++ b/tests/Stub/UserModelStub.php @@ -0,0 +1,67 @@ + array, 'send_email' => bool] + * + * @var array + */ + public array $aCreated = []; + + /** + * The ID to give the next account; incremented per call + */ + public int $iNextId = 100; + + /** + * What create() should do instead of succeeding: 'false' to report a + * failure the way the real model does, 'throw' to fall over + */ + public ?string $sFail = null; + + /** + * The error create() should leave behind when it returns false + */ + public ?string $sCreateError = 'The account could not be created'; + + // -------------------------------------------------------------------------- + + public function create(array $data = [], $bSendWelcome = true) + { + $this->aCreated[] = [ + 'data' => $data, + 'send_email' => (bool) $bSendWelcome, + ]; + + if ($this->sFail === 'throw') { + throw new RuntimeException('the account could not be written'); + + } elseif ($this->sFail === 'false') { + + if ($this->sCreateError !== null) { + $this->setError($this->sCreateError); + } + + return false; + } + + return new Resource\User((object) [ + 'id' => $this->iNextId++, + 'email' => $data['email'] ?? null, + ]); + } +} diff --git a/tests/Stub/ValidatorWithStub.php b/tests/Stub/ValidatorWithStub.php new file mode 100644 index 00000000..cc962864 --- /dev/null +++ b/tests/Stub/ValidatorWithStub.php @@ -0,0 +1,24 @@ +oImportService; + } +} diff --git a/tests/Validator/User/IdentifierTest.php b/tests/Validator/User/IdentifierTest.php new file mode 100644 index 00000000..a484c913 --- /dev/null +++ b/tests/Validator/User/IdentifierTest.php @@ -0,0 +1,37 @@ +run($aData); + return []; + } catch (ValidationException $e) { + return $e->getData(); + } + } + + public function test_the_identifier_must_be_an_email_when_logging_in_by_email(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + self::assertSame(['identifier' => 'This must be a valid email.'], $this->errorsFor(['identifier' => 'ada'])); + self::assertSame([], $this->errorsFor(['identifier' => 'ada@example.com'])); + } + + public function test_any_value_will_do_otherwise(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'BOTH'); + + self::assertSame(['identifier' => 'This field is required.'], $this->errorsFor(['identifier' => ''])); + self::assertSame([], $this->errorsFor(['identifier' => 'ada'])); + } +} diff --git a/tests/Validator/User/IdentityTest.php b/tests/Validator/User/IdentityTest.php new file mode 100644 index 00000000..a505b6de --- /dev/null +++ b/tests/Validator/User/IdentityTest.php @@ -0,0 +1,123 @@ + taken values) + */ + private function validator(array $aTaken = [], ?int $iIgnoreUserId = null): Identity + { + return (new Identity($iIgnoreUserId)) + ->stubRule(FormValidation::RULE_IS_UNIQUE, function ($mValue, Context $oContext) use ($aTaken) { + [, $sColumn] = $oContext->getParams(); + return !in_array($mValue, $aTaken[$sColumn] ?? [], true); + }); + } + + private function errorsFor(Identity $oValidator, array $aData): array + { + try { + $oValidator->run($aData); + return []; + } catch (ValidationException $e) { + return $e->getData(); + } + } + + // -------------------------------------------------------------------------- + + public function test_both_fields_are_required_when_logging_in_with_both(): void + { + self::assertSame( + ['email', 'username'], + array_keys($this->errorsFor($this->validator(), [])) + ); + } + + public function test_a_clean_identity_passes_and_is_trimmed(): void + { + $oValidator = $this->validator(); + $oValidator->run(['email' => ' ada@example.com ', 'username' => ' ada.lovelace ']); + + self::assertSame( + ['email' => 'ada@example.com', 'username' => 'ada.lovelace'], + $oValidator->getValidatedData() + ); + } + + public function test_a_taken_identity_reports_the_already_registered_message(): void + { + $aErrors = $this->errorsFor( + $this->validator(['email' => ['ada@example.com']]), + ['email' => 'ada@example.com', 'username' => 'ada'] + ); + + self::assertSame(['email'], array_keys($aErrors)); + self::assertStringContainsString('already registered', $aErrors['email']); + } + + public function test_the_ignored_user_is_passed_to_the_uniqueness_rule(): void + { + $aSeen = []; + $oValidator = (new Identity(42)) + ->stubRule(FormValidation::RULE_IS_UNIQUE, function ($mValue, Context $oContext) use (&$aSeen) { + $aSeen[$oContext->getField()] = $oContext->getParams(); + return true; + }); + + $oValidator->run(['email' => 'ada@example.com', 'username' => 'ada']); + + self::assertSame(['42', 'user_id'], array_slice($aSeen['email'], 2)); + self::assertSame(['42', 'id'], array_slice($aSeen['username'], 2)); + } + + public function test_malformed_values_fail_before_uniqueness_is_checked(): void + { + $aErrors = $this->errorsFor( + $this->validator(), + ['email' => 'not-an-email', 'username' => 'has spaces!'] + ); + + self::assertSame('This must be a valid email.', $aErrors['email']); + self::assertStringContainsString('alpha-numeric', $aErrors['username']); + } + + public function test_only_the_configured_identity_is_validated(): void + { + Config::set('APP_NATIVE_LOGIN_USING', 'EMAIL'); + + self::assertSame(['email'], array_keys($this->validator()->getRules())); + self::assertSame([], $this->errorsFor($this->validator(), ['email' => 'ada@example.com'])); + } + + public function test_a_field_can_be_switched_off_by_overriding_it_with_no_rules(): void + { + $oValidator = $this->validator()->setRules(['username' => []]); + + self::assertSame([], $this->errorsFor($oValidator, ['email' => 'ada@example.com'])); + } + + public function test_extra_rules_merge_with_the_identity_rules(): void + { + $oValidator = $this->validator()->addRules(['first_name' => [FormValidation::RULE_REQUIRED]]); + + self::assertSame( + ['email', 'username', 'first_name'], + array_keys($this->errorsFor($oValidator, [])) + ); + } +} diff --git a/tests/bootstrap.php b/tests/bootstrap.php index fd349ea2..93ac78fe 100644 --- a/tests/bootstrap.php +++ b/tests/bootstrap.php @@ -1,3 +1,7 @@