diff --git a/composer.json b/composer.json index 99d92ff5..c5b373a9 100644 --- a/composer.json +++ b/composer.json @@ -38,6 +38,7 @@ "nails/module-cron": "dev-develop", "nails/module-email": "dev-develop", "nails/module-form-builder": "dev-develop", + "nails/module-housekeeping": "dev-develop", "sonata-project/google-authenticator": "~2.3.0", "wikimedia/common-passwords": "^v0.5", "lbuchs/webauthn": "^2.2", diff --git a/src/Api/Controller/Import.php b/src/Api/Controller/Import.php index 90c55b96..6f3c0642 100644 --- a/src/Api/Controller/Import.php +++ b/src/Api/Controller/Import.php @@ -242,7 +242,7 @@ protected function delete(ApiResponse $oApiResponse, Entity $oItem): void * * A failure here is logged rather than raised: the row has gone and the * delete has genuinely succeeded, so there is nothing for the caller to - * retry. It cannot be left silent though - `auth:user:import:clean` + * retry. It cannot be left silent though - `Nails\Auth\Housekeeping\UserImports` * reaps by walking job rows, so an object orphaned here is unreachable * by anything else, and the log line is the only thread back to what it * was for. diff --git a/src/Console/Command/User/Import/Clean.php b/src/Console/Command/User/Import/Clean.php deleted file mode 100644 index 4b8d3afc..00000000 --- a/src/Console/Command/User/Import/Clean.php +++ /dev/null @@ -1,261 +0,0 @@ -setName('auth:user:import:clean') - ->setDescription('Releases orphaned user import jobs and reaps old ones'); - } - - // -------------------------------------------------------------------------- - - /** - * Executes the command - * - * @param InputInterface $oInput The Input Interface provided by Symfony - * @param OutputInterface $oOutput The Output Interface provided by Symfony - */ - protected function execute(InputInterface $oInput, OutputInterface $oOutput): int - { - parent::execute($oInput, $oOutput); - - try { - - $this->banner('User Import: Clean'); - $this - ->releaseOrphans() - ->reapDrafts() - ->rotateFinished(); - - } catch (Throwable $e) { - return $this->abort( - self::EXIT_CODE_FAILURE, - [$e->getMessage()] - ); - } - - $oOutput->writeln(''); - $oOutput->writeln('Complete!'); - - return self::EXIT_CODE_SUCCESS; - } - - // -------------------------------------------------------------------------- - - /** - * Releases claims held by processes which are no longer with us - * - * The status is deliberately left alone; a job resumes from its cursor, and - * sending it back to PENDING would restart it from the top and re-validate - * rows whose users have since been created. - * - * @throws FactoryException - * @throws ModelException - */ - protected function releaseOrphans(): self - { - $this->oOutput->writeln('Releasing orphaned claims'); - - /** @var Model\User\Import $oModel */ - $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); - /** @var Database $oDb */ - $oDb = Factory::service('Database'); - - $oDb - ->set('claim_token', null) - ->set('claimed', null) - ->where('claim_token IS NOT NULL', null, false) - ->where('claimed <', $this->getCutOff('AUTH_USER_IMPORT_STALE_CLAIM', static::STALE_CLAIM)) - ->update($oModel->getTableName()); - - $this->oOutput->writeln(sprintf( - 'Released %s', - $oDb->affected_rows() - )); - - return $this; - } - - // -------------------------------------------------------------------------- - - /** - * Deletes uploads which were never approved - * - * @throws FactoryException - * @throws ModelException - */ - protected function reapDrafts(): self - { - $this->oOutput->writeln(''); - $this->oOutput->writeln('Reaping abandoned drafts'); - - $iDeleted = $this->deleteJobs( - [Status::DRAFT], - $this->getCutOff('AUTH_USER_IMPORT_DRAFT_TTL', static::DRAFT_TTL) - ); - - $this->oOutput->writeln(sprintf('Deleted %s', $iDeleted)); - - return $this; - } - - // -------------------------------------------------------------------------- - - /** - * Deletes jobs which finished long enough ago that nobody is coming back for them - * - * @throws FactoryException - * @throws ModelException - */ - protected function rotateFinished(): self - { - $this->oOutput->writeln(''); - $this->oOutput->writeln('Rotating finished jobs'); - - $iDeleted = $this->deleteJobs( - Status::terminal(), - $this->getCutOff('AUTH_USER_IMPORT_RETENTION', static::RETENTION) - ); - - $this->oOutput->writeln(sprintf('Deleted %s', $iDeleted)); - - return $this; - } - - // -------------------------------------------------------------------------- - - /** - * Deletes jobs of the given statuses which were last modified before the cut off - * - * @param Status[] $aStatuses - * - * @throws FactoryException - * @throws ModelException - */ - protected function deleteJobs(array $aStatuses, string $sCutOff): int - { - /** @var Model\User\Import $oModel */ - $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); - /** @var Database $oDb */ - $oDb = Factory::service('Database'); - - $aRows = $oDb - ->select('id') - ->where_in('status', Status::values($aStatuses)) - ->where('modified <', $sCutOff) - ->order_by('id', 'asc') - ->limit(static::MAX_PER_RUN) - ->get($oModel->getTableName()) - ->result(); - - $iDeleted = 0; - - foreach ($aRows as $oRow) { - - /** @var Resource\User\Import|null $oImport */ - $oImport = $oModel->getById((int) $oRow->id); - if (empty($oImport)) { - continue; - } - - // The job goes first; the CDN objects cascade onto it, and a - // half-deleted job is worse than a lingering file. - if (!$oModel->delete($oImport->id)) { - $this->oOutput->writeln(sprintf( - '↳ Failed to delete import #%s; %s', - $oImport->id, - $oModel->lastError() - )); - continue; - } - - foreach ($oModel->destroyObjects($oImport) as $iObjectId => $sError) { - $this->oOutput->writeln(sprintf( - '↳ Failed to destroy CDN object #%s; %s', - $iObjectId, - $sError - )); - } - - $iDeleted++; - } - - return $iDeleted; - } - - // -------------------------------------------------------------------------- - - /** - * Returns the datetime $sConfigKey seconds ago - * - * @throws FactoryException - */ - protected function getCutOff(string $sConfigKey, int $iDefault): string - { - $iSeconds = (int) Config::get($sConfigKey, $iDefault) ?: $iDefault; - - /** @var \DateTime $oCutOff */ - $oCutOff = Factory::factory('DateTime'); - $oCutOff->sub(new DateInterval('PT' . $iSeconds . 'S')); - - return $oCutOff->format('Y-m-d H:i:s'); - } -} diff --git a/src/Cron/Task/User/Import/Clean.php b/src/Cron/Task/User/Import/Clean.php deleted file mode 100644 index 6aec2a25..00000000 --- a/src/Cron/Task/User/Import/Clean.php +++ /dev/null @@ -1,39 +0,0 @@ - + */ + protected function where(): array + { + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + + return [ + ['expires <', $oNow->format('Y-m-d H:i:s')], + ]; + } + + /** + * @return string[] + */ + protected function auditColumns(): array + { + return ['id', 'user_id', 'expires']; + } + + protected function optimizeAfter(): bool + { + return true; + } +} diff --git a/src/Housekeeping/TwoFactorTokens.php b/src/Housekeeping/TwoFactorTokens.php new file mode 100644 index 00000000..4db0310d --- /dev/null +++ b/src/Housekeeping/TwoFactorTokens.php @@ -0,0 +1,87 @@ +format('Y-m-d H:i:s'); + $iBatchSize = 200; + $iProcessed = 0; + $iLastId = 0; + + $oContext + ->writeln(sprintf('Deleting from %s in batches of %d', $sTable, $iBatchSize)) + ->log(sprintf( + 'TABLE %s batch_size=%d dry_run=%s', + $sTable, + $iBatchSize, + $oContext->isDryRun() ? 'true' : 'false' + )); + + while (true) { + $aRows = $oDb + ->select('id, user_id, expires') + ->where('expires <', $sCutOff) + ->where('id >', $iLastId) + ->order_by('id', 'asc') + ->limit($iBatchSize) + ->get($sTable) + ->result(); + + if (empty($aRows)) { + break; + } + + $aIds = []; + foreach ($aRows as $oRow) { + $iId = (int) $oRow->id; + $iLastId = $iId; + $aIds[] = $iId; + $sAudit = sprintf( + 'id=%d user_id=%s expires=%s', + $iId, + $oRow->user_id === null ? 'null' : (string) $oRow->user_id, + (string) $oRow->expires + ); + $oContext + ->log('DELETE ' . $sAudit) + ->writeln(' ↳ ' . $sAudit); + } + + if (!$oContext->isDryRun()) { + $oDb + ->where_in('id', $aIds) + ->delete($sTable); + } + + $iProcessed += count($aIds); + } + + $oContext->writeln(sprintf( + '%s %s', + number_format($iProcessed), + $oContext->isDryRun() ? 'would be deleted' : 'deleted' + )); + + return Result::ok($iProcessed); + } +} diff --git a/src/Housekeeping/UserEvents.php b/src/Housekeeping/UserEvents.php new file mode 100644 index 00000000..56534e78 --- /dev/null +++ b/src/Housekeeping/UserEvents.php @@ -0,0 +1,81 @@ + + */ + protected function where(): array + { + $iDays = $this->retentionDays(); + if ($iDays < 1) { + return [['id' => 0]]; + } + + /** @var \DateTime $oNow */ + $oNow = Factory::factory('DateTime'); + $oNow->sub(new \DateInterval('P' . $iDays . 'D')); + + return [ + ['created <', $oNow->format('Y-m-d H:i:s')], + ]; + } + + /** + * @return string[] + */ + protected function auditColumns(): array + { + return ['id', 'created_by', 'type', 'created']; + } + + protected function optimizeAfter(): bool + { + return true; + } + + public function execute(Context $oContext): Result + { + $iDays = $this->retentionDays(); + if ($iDays < 1) { + $oContext + ->writeln('User event cleanup disabled') + ->log('DISABLED AUTH_USER_EVENT_RETENTION_DAYS=0'); + + return Result::ok(0, 'User event cleanup disabled'); + } + + $oContext->writeln('Retention policy: ' . $iDays . ' days'); + + return $this->deleteModelRows($oContext); + } + + protected function retentionDays(): int + { + return (int) Config::get('AUTH_USER_EVENT_RETENTION_DAYS', 0); + } +} diff --git a/src/Housekeeping/UserImports.php b/src/Housekeeping/UserImports.php new file mode 100644 index 00000000..81b0dd01 --- /dev/null +++ b/src/Housekeeping/UserImports.php @@ -0,0 +1,255 @@ +releaseOrphans($oContext); + $iDrafts = $this->reapDrafts($oContext); + $iFinished = $this->rotateFinished($oContext); + + return Result::ok($iReleased + $iDrafts + $iFinished); + } + + /** + * Releases claims held by processes which are no longer with us. + * + * The status is deliberately left alone; a job resumes from its cursor, and + * sending it back to PENDING would restart it from the top and re-validate + * rows whose users have since been created. + */ + protected function releaseOrphans(Context $oContext): int + { + $oContext->writeln('Releasing orphaned claims'); + + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $sCutOff = $this->getCutOff('AUTH_USER_IMPORT_STALE_CLAIM', static::STALE_CLAIM); + $iProcessed = 0; + $iLastId = 0; + + while (true) { + $aRows = $oDb + ->select('id, status, modified, claimed') + ->where('claim_token IS NOT NULL', null, false) + ->where('claimed <', $sCutOff) + ->where('id >', $iLastId) + ->order_by('id', 'asc') + ->limit(200) + ->get($oModel->getTableName()) + ->result(); + + if (empty($aRows)) { + break; + } + + $aIds = []; + foreach ($aRows as $oRow) { + $iId = (int) $oRow->id; + $iLastId = $iId; + $aIds[] = $iId; + $sAudit = sprintf( + 'id=%d status=%s modified=%s claimed=%s', + $iId, + (string) $oRow->status, + (string) $oRow->modified, + (string) $oRow->claimed + ); + $oContext + ->log('RELEASE ' . $sAudit) + ->writeln(' ↳ ' . $sAudit); + } + + if (!$oContext->isDryRun()) { + $oDb + ->set('claim_token', null) + ->set('claimed', null) + ->where_in('id', $aIds) + ->update($oModel->getTableName()); + } + + $iProcessed += count($aIds); + } + + $oContext->writeln(sprintf( + 'Released %s', + number_format($iProcessed) + )); + + return $iProcessed; + } + + protected function reapDrafts(Context $oContext): int + { + $oContext->writeln(''); + $oContext->writeln('Reaping abandoned drafts'); + + $iDeleted = $this->deleteJobs( + $oContext, + [Status::DRAFT], + $this->getCutOff('AUTH_USER_IMPORT_DRAFT_TTL', static::DRAFT_TTL) + ); + + $oContext->writeln(sprintf('Deleted %s', number_format($iDeleted))); + + return $iDeleted; + } + + protected function rotateFinished(Context $oContext): int + { + $oContext->writeln(''); + $oContext->writeln('Rotating finished jobs'); + + $iDeleted = $this->deleteJobs( + $oContext, + Status::terminal(), + $this->getCutOff('AUTH_USER_IMPORT_RETENTION', static::RETENTION) + ); + + $oContext->writeln(sprintf('Deleted %s', number_format($iDeleted))); + + return $iDeleted; + } + + /** + * @param Status[] $aStatuses + */ + protected function deleteJobs(Context $oContext, array $aStatuses, string $sCutOff): int + { + /** @var Model\User\Import $oModel */ + $oModel = Factory::model('UserImport', Constants::MODULE_SLUG); + /** @var Database $oDb */ + $oDb = Factory::service('Database'); + + $aRows = $oDb + ->select('id') + ->where_in('status', Status::values($aStatuses)) + ->where('modified <', $sCutOff) + ->order_by('id', 'asc') + ->limit(static::MAX_PER_RUN) + ->get($oModel->getTableName()) + ->result(); + + $iDeleted = 0; + + foreach ($aRows as $oRow) { + /** @var Resource\User\Import|null $oImport */ + $oImport = $oModel->getById((int) $oRow->id); + if (empty($oImport)) { + continue; + } + + $sStatus = $oImport->status instanceof Status + ? $oImport->status->value + : (string) $oImport->status; + $sAudit = sprintf( + 'id=%d status=%s modified=%s', + (int) $oImport->id, + $sStatus, + $this->stringifyDate($oImport->modified) + ); + + $oContext + ->log('DELETE ' . $sAudit) + ->writeln(' ↳ ' . $sAudit); + + if ($oContext->isDryRun()) { + $iDeleted++; + continue; + } + + // The job goes first; the CDN objects cascade onto it, and a + // half-deleted job is worse than a lingering file. + if (!$oModel->delete($oImport->id)) { + $oContext + ->log('ERROR id=' . $oImport->id . ' ' . $oModel->lastError()) + ->writeln(sprintf( + '↳ Failed to delete import #%s; %s', + $oImport->id, + $oModel->lastError() + )); + continue; + } + + foreach ($oModel->destroyObjects($oImport) as $iObjectId => $sError) { + $oContext + ->log('ERROR cdn_object=' . $iObjectId . ' ' . $sError) + ->writeln(sprintf( + '↳ Failed to destroy CDN object #%s; %s', + $iObjectId, + $sError + )); + } + + $iDeleted++; + } + + return $iDeleted; + } + + protected function getCutOff(string $sConfigKey, int $iDefault): string + { + $iSeconds = (int) Config::get($sConfigKey, $iDefault) ?: $iDefault; + + /** @var \DateTime $oCutOff */ + $oCutOff = Factory::factory('DateTime'); + $oCutOff->sub(new DateInterval('PT' . $iSeconds . 'S')); + + return $oCutOff->format('Y-m-d H:i:s'); + } + + protected function stringifyDate(mixed $mValue): string + { + if ($mValue === null) { + return 'null'; + } + + if ($mValue instanceof \DateTimeInterface) { + return $mValue->format('Y-m-d H:i:s'); + } + + return (string) $mValue; + } +} diff --git a/src/Model/User/Import.php b/src/Model/User/Import.php index 7d8d13e6..1179e0fa 100644 --- a/src/Model/User/Import.php +++ b/src/Model/User/Import.php @@ -224,7 +224,7 @@ protected function countItems(int $iId, ?ItemStatus $oStatus = null): int * * Failures are collected rather than thrown: the row is already gone, so a * file which outlives it is litter, not an error. Note that - * `auth:user:import:clean` walks job rows, so it will never reap these. + * `Nails\Auth\Housekeeping\UserImports` walks job rows, so it will never reap these. * objectDestroy() signals failure both by returning false - for a missing * object, a driver failure, or a rolled back transaction - and by throwing, * so both are handled. diff --git a/src/Service/User/Import/Processor.php b/src/Service/User/Import/Processor.php index fd75d25b..d5fb53f9 100644 --- a/src/Service/User/Import/Processor.php +++ b/src/Service/User/Import/Processor.php @@ -1103,7 +1103,7 @@ protected function clearPreviousAttempt(Resource\User\Import $oImport): void } /** - * begin() is about to null log_id, and auth:user:import:clean only reaps + * begin() is about to null log_id, and UserImports housekeeping only reaps * objects it can still reach from a job row, so the previous log is * destroyed here or not at all. */