From 7847244c076feaf9e60a44acc4f426c3eb6babef Mon Sep 17 00:00:00 2001 From: 128Na Date: Thu, 20 Aug 2026 19:56:27 +0900 Subject: [PATCH 1/4] fix(scrape): sleep after failure too, back off longer on 429 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Japan/Twitrans の Handler は Sleep::for(1)->second() が try 節の成功 パスにしかなく、失敗時は間隔を空けずに次の URL へ進んでいた。 wikiwiki.jp (Twitrans) で一度 429 を受けると、間隔なしの連続アクセス がそのまま続き、以降ほぼ全URLが 429 で失敗する連鎖が起きていた (prod 8/19 ログで286件失敗)。 wikiwiki.jp 側に公開されたレート制限値は無い(robots.txt に Crawl-delay 指定なし、Cloudflare 経由)ため、固定の「安全な間隔」を 決め打ちするのではなく、失敗時にも通常間隔(2秒)を必ず空け、429を 受けた直後だけ長めのクールダウン(15秒)を挟むようにした。 Co-Authored-By: Claude Sonnet 5 --- app/Actions/Scrape/Japan/Handler.php | 20 +++++++- app/Actions/Scrape/Twitrans/Handler.php | 20 +++++++- .../Scrape/Twitrans/HandlerRateLimitTest.php | 48 +++++++++++++++++++ 3 files changed, 86 insertions(+), 2 deletions(-) create mode 100644 tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php diff --git a/app/Actions/Scrape/Japan/Handler.php b/app/Actions/Scrape/Japan/Handler.php index 875ea90..b6430ca 100644 --- a/app/Actions/Scrape/Japan/Handler.php +++ b/app/Actions/Scrape/Japan/Handler.php @@ -9,11 +9,16 @@ use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Enums\Encoding; use App\Enums\SiteName; +use Illuminate\Http\Client\RequestException; use Illuminate\Support\Sleep; use Psr\Log\LoggerInterface; final readonly class Handler implements HandlerInterface { + private const int IntervalSeconds = 2; + + private const int RateLimitCooldownSeconds = 15; + public function __construct( private FetchHtml $fetchHtml, private FindUrls $findUrls, @@ -34,10 +39,23 @@ public function __invoke(LoggerInterface $logger): void SiteName::Japan, $html ); - Sleep::for(1)->second(); + Sleep::for(self::IntervalSeconds)->seconds(); } catch (\Throwable $th) { $logger->error('failed', [$url, $th]); + $this->sleepAfterFailure($th); } } } + + private function sleepAfterFailure(\Throwable $th): void + { + // 429 は通常のリトライ間隔では解消しないため、長めに待って次の URL へ進む。 + if ($th instanceof RequestException && $th->response->status() === 429) { + Sleep::for(self::RateLimitCooldownSeconds)->seconds(); + + return; + } + + Sleep::for(self::IntervalSeconds)->seconds(); + } } diff --git a/app/Actions/Scrape/Twitrans/Handler.php b/app/Actions/Scrape/Twitrans/Handler.php index dc641cd..1da2f16 100644 --- a/app/Actions/Scrape/Twitrans/Handler.php +++ b/app/Actions/Scrape/Twitrans/Handler.php @@ -9,11 +9,16 @@ use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Enums\Encoding; use App\Enums\SiteName; +use Illuminate\Http\Client\RequestException; use Illuminate\Support\Sleep; use Psr\Log\LoggerInterface; final readonly class Handler implements HandlerInterface { + private const int IntervalSeconds = 2; + + private const int RateLimitCooldownSeconds = 15; + public function __construct( private FetchHtml $fetchHtml, private FindUrls $findUrls, @@ -34,10 +39,23 @@ public function __invoke(LoggerInterface $logger): void SiteName::Twitrans, $html ); - Sleep::for(1)->second(); + Sleep::for(self::IntervalSeconds)->seconds(); } catch (\Throwable $th) { $logger->error('failed', [$url, $th]); + $this->sleepAfterFailure($th); } } } + + private function sleepAfterFailure(\Throwable $th): void + { + // 429 は通常のリトライ間隔では解消しないため、長めに待って次の URL へ進む。 + if ($th instanceof RequestException && $th->response->status() === 429) { + Sleep::for(self::RateLimitCooldownSeconds)->seconds(); + + return; + } + + Sleep::for(self::IntervalSeconds)->seconds(); + } } diff --git a/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php b/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php new file mode 100644 index 0000000..7fffab8 --- /dev/null +++ b/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php @@ -0,0 +1,48 @@ +
'; + + Http::fake([ + 'https://wikiwiki.jp/twitrans?cmd=list' => Http::response($listHtml, 200), + 'https://wikiwiki.jp/twitrans/addon/pak128.japan/Ok' => Http::response('ok', 200), + 'https://wikiwiki.jp/twitrans/addon/pak128.japan/Limited' => Http::response('rate limited', 429), + ]); + + $handler = new Handler( + new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false), + new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)), + new UpdateOrCreateRawPage, + ); + + $handler(new NullLogger); + + // 成功時は通常間隔(2秒)、429 を受けた直後はより長いクールダウン(15秒)。 + Sleep::assertSequence([ + Sleep::for(2)->seconds(), + Sleep::for(15)->seconds(), + ]); + } +} From f66134e5b2d1b2a267684c3e40aa9bef4a844509 Mon Sep 17 00:00:00 2001 From: GitHub Action Date: Thu, 20 Aug 2026 10:57:07 +0000 Subject: [PATCH 2/4] [rector] Rector fixes --- app/Actions/Scrape/Japan/Handler.php | 4 ++-- app/Actions/Scrape/Twitrans/Handler.php | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/app/Actions/Scrape/Japan/Handler.php b/app/Actions/Scrape/Japan/Handler.php index b6430ca..9942b91 100644 --- a/app/Actions/Scrape/Japan/Handler.php +++ b/app/Actions/Scrape/Japan/Handler.php @@ -47,10 +47,10 @@ public function __invoke(LoggerInterface $logger): void } } - private function sleepAfterFailure(\Throwable $th): void + private function sleepAfterFailure(\Throwable $throwable): void { // 429 は通常のリトライ間隔では解消しないため、長めに待って次の URL へ進む。 - if ($th instanceof RequestException && $th->response->status() === 429) { + if ($throwable instanceof RequestException && $throwable->response->status() === 429) { Sleep::for(self::RateLimitCooldownSeconds)->seconds(); return; diff --git a/app/Actions/Scrape/Twitrans/Handler.php b/app/Actions/Scrape/Twitrans/Handler.php index 1da2f16..4f4d55d 100644 --- a/app/Actions/Scrape/Twitrans/Handler.php +++ b/app/Actions/Scrape/Twitrans/Handler.php @@ -47,10 +47,10 @@ public function __invoke(LoggerInterface $logger): void } } - private function sleepAfterFailure(\Throwable $th): void + private function sleepAfterFailure(\Throwable $throwable): void { // 429 は通常のリトライ間隔では解消しないため、長めに待って次の URL へ進む。 - if ($th instanceof RequestException && $th->response->status() === 429) { + if ($throwable instanceof RequestException && $throwable->response->status() === 429) { Sleep::for(self::RateLimitCooldownSeconds)->seconds(); return; From f3531503d901347b2b09cebe710f325c76a3e7ce Mon Sep 17 00:00:00 2001 From: 128Na Date: Thu, 20 Aug 2026 20:04:41 +0900 Subject: [PATCH 3/4] fix(scrape): raise Twitrans interval/cooldown after live verification MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ローカルから php artisan app:scrape twitrans を実際に実行して検証した ところ、2秒間隔+429後15秒クールダウンでは不十分だった。最初の4件は バーストで通るが、それ以降は2秒間隔だとトークンが回復しきらず断続的 に429になり、15秒待っても直後の1件しか通らない(トークンバケット的な 挙動)。 Twitrans (wikiwiki.jp) のみ通常間隔を10秒、429後クールダウンを60秒に 引き上げる。Japan (japanese.simutrans.com) 側は429が一度も観測されて いないため据え置き。 Co-Authored-By: Claude Sonnet 5 --- app/Actions/Scrape/Twitrans/Handler.php | 4 ++-- .../Actions/Scrape/Twitrans/HandlerRateLimitTest.php | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/Actions/Scrape/Twitrans/Handler.php b/app/Actions/Scrape/Twitrans/Handler.php index 4f4d55d..1699842 100644 --- a/app/Actions/Scrape/Twitrans/Handler.php +++ b/app/Actions/Scrape/Twitrans/Handler.php @@ -15,9 +15,9 @@ final readonly class Handler implements HandlerInterface { - private const int IntervalSeconds = 2; + private const int IntervalSeconds = 10; - private const int RateLimitCooldownSeconds = 15; + private const int RateLimitCooldownSeconds = 60; public function __construct( private FetchHtml $fetchHtml, diff --git a/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php b/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php index 7fffab8..afeff2a 100644 --- a/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php +++ b/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php @@ -39,10 +39,10 @@ public function test_backs_off_longer_after_429_than_after_success(): void $handler(new NullLogger); - // 成功時は通常間隔(2秒)、429 を受けた直後はより長いクールダウン(15秒)。 + // 成功時は通常間隔(10秒)、429 を受けた直後はより長いクールダウン(60秒)。 Sleep::assertSequence([ - Sleep::for(2)->seconds(), - Sleep::for(15)->seconds(), + Sleep::for(10)->seconds(), + Sleep::for(60)->seconds(), ]); } } From 3a1d9903999545e2d54b1ed0684fe74c3440a781 Mon Sep 17 00:00:00 2001 From: 128Na Date: Thu, 20 Aug 2026 20:32:01 +0900 Subject: [PATCH 4/4] refactor(scrape): extract shared SleepBetweenRequests, fake Sleep in tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit コードレビュー指摘への対応。 - Japan/Twitrans の Handler に一字一句同じ sleepAfterFailure() が重複 していたため、App\Actions\Scrape\SleepBetweenRequests に抽出。間隔/ クールダウン秒数は各 Handler の定数のまま呼び出し時に渡す形にし、 サイトごとの値の違い(Japan 2/15秒、Twitrans 10/60秒)は維持。 あわせて定数名を IntervalSeconds → INTERVAL_SECONDS 等、同ディレク トリの既存定数(TOP_URL 等)に合わせた UPPER_SNAKE_CASE に統一。 - Scrape/Japan の HandlerFailureTest・HandlerIsolationTest が Sleep::fake() していなかったため、失敗時にも sleep するようになった 前回PRの変更で実際に数秒待つようになっていた。Sleep::fake() を追加 して解消(該当テストの実行時間 ~8秒 → ~0.1秒)。 Co-Authored-By: Claude Sonnet 5 --- app/Actions/Scrape/Japan/Handler.php | 24 +++++-------------- app/Actions/Scrape/SleepBetweenRequests.php | 23 ++++++++++++++++++ app/Actions/Scrape/Twitrans/Handler.php | 24 +++++-------------- .../Scrape/Japan/HandlerFailureTest.php | 6 +++++ .../Scrape/Japan/HandlerIsolationTest.php | 4 ++++ .../Scrape/Twitrans/HandlerRateLimitTest.php | 2 ++ 6 files changed, 47 insertions(+), 36 deletions(-) create mode 100644 app/Actions/Scrape/SleepBetweenRequests.php diff --git a/app/Actions/Scrape/Japan/Handler.php b/app/Actions/Scrape/Japan/Handler.php index 9942b91..a14963c 100644 --- a/app/Actions/Scrape/Japan/Handler.php +++ b/app/Actions/Scrape/Japan/Handler.php @@ -6,23 +6,23 @@ use App\Actions\Scrape\FetchHtml; use App\Actions\Scrape\HandlerInterface; +use App\Actions\Scrape\SleepBetweenRequests; use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Enums\Encoding; use App\Enums\SiteName; -use Illuminate\Http\Client\RequestException; -use Illuminate\Support\Sleep; use Psr\Log\LoggerInterface; final readonly class Handler implements HandlerInterface { - private const int IntervalSeconds = 2; + private const int INTERVAL_SECONDS = 2; - private const int RateLimitCooldownSeconds = 15; + private const int RATE_LIMIT_COOLDOWN_SECONDS = 15; public function __construct( private FetchHtml $fetchHtml, private FindUrls $findUrls, private UpdateOrCreateRawPage $updateOrCreateRawPage, + private SleepBetweenRequests $sleepBetweenRequests, ) {} #[\Override] @@ -39,23 +39,11 @@ public function __invoke(LoggerInterface $logger): void SiteName::Japan, $html ); - Sleep::for(self::IntervalSeconds)->seconds(); + ($this->sleepBetweenRequests)(null, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS); } catch (\Throwable $th) { $logger->error('failed', [$url, $th]); - $this->sleepAfterFailure($th); + ($this->sleepBetweenRequests)($th, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS); } } } - - private function sleepAfterFailure(\Throwable $throwable): void - { - // 429 は通常のリトライ間隔では解消しないため、長めに待って次の URL へ進む。 - if ($throwable instanceof RequestException && $throwable->response->status() === 429) { - Sleep::for(self::RateLimitCooldownSeconds)->seconds(); - - return; - } - - Sleep::for(self::IntervalSeconds)->seconds(); - } } diff --git a/app/Actions/Scrape/SleepBetweenRequests.php b/app/Actions/Scrape/SleepBetweenRequests.php new file mode 100644 index 0000000..7a22d50 --- /dev/null +++ b/app/Actions/Scrape/SleepBetweenRequests.php @@ -0,0 +1,23 @@ +response->status() === 429) { + Sleep::for($rateLimitCooldownSeconds)->seconds(); + + return; + } + + Sleep::for($intervalSeconds)->seconds(); + } +} diff --git a/app/Actions/Scrape/Twitrans/Handler.php b/app/Actions/Scrape/Twitrans/Handler.php index 1699842..3c27b93 100644 --- a/app/Actions/Scrape/Twitrans/Handler.php +++ b/app/Actions/Scrape/Twitrans/Handler.php @@ -6,23 +6,23 @@ use App\Actions\Scrape\FetchHtml; use App\Actions\Scrape\HandlerInterface; +use App\Actions\Scrape\SleepBetweenRequests; use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Enums\Encoding; use App\Enums\SiteName; -use Illuminate\Http\Client\RequestException; -use Illuminate\Support\Sleep; use Psr\Log\LoggerInterface; final readonly class Handler implements HandlerInterface { - private const int IntervalSeconds = 10; + private const int INTERVAL_SECONDS = 10; - private const int RateLimitCooldownSeconds = 60; + private const int RATE_LIMIT_COOLDOWN_SECONDS = 60; public function __construct( private FetchHtml $fetchHtml, private FindUrls $findUrls, private UpdateOrCreateRawPage $updateOrCreateRawPage, + private SleepBetweenRequests $sleepBetweenRequests, ) {} #[\Override] @@ -39,23 +39,11 @@ public function __invoke(LoggerInterface $logger): void SiteName::Twitrans, $html ); - Sleep::for(self::IntervalSeconds)->seconds(); + ($this->sleepBetweenRequests)(null, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS); } catch (\Throwable $th) { $logger->error('failed', [$url, $th]); - $this->sleepAfterFailure($th); + ($this->sleepBetweenRequests)($th, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS); } } } - - private function sleepAfterFailure(\Throwable $throwable): void - { - // 429 は通常のリトライ間隔では解消しないため、長めに待って次の URL へ進む。 - if ($throwable instanceof RequestException && $throwable->response->status() === 429) { - Sleep::for(self::RateLimitCooldownSeconds)->seconds(); - - return; - } - - Sleep::for(self::IntervalSeconds)->seconds(); - } } diff --git a/tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php b/tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php index 789c447..083fbc4 100644 --- a/tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php +++ b/tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php @@ -7,10 +7,12 @@ use App\Actions\Scrape\FetchHtml; use App\Actions\Scrape\Japan\FindUrls; use App\Actions\Scrape\Japan\Handler; +use App\Actions\Scrape\SleepBetweenRequests; use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Models\RawPage; use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Sleep; use Psr\Log\NullLogger; use Tests\Feature\TestCase; @@ -23,6 +25,7 @@ final class HandlerFailureTest extends TestCase public function test_does_not_write_raw_page_when_fetch_fails(): void { Http::preventStrayRequests(); + Sleep::fake(); $listHtml = '
  • ' .'test' @@ -39,6 +42,7 @@ public function test_does_not_write_raw_page_when_fetch_fails(): void new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false), new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)), new UpdateOrCreateRawPage, + new SleepBetweenRequests, ); $handler(new NullLogger); @@ -49,6 +53,7 @@ public function test_does_not_write_raw_page_when_fetch_fails(): void public function test_does_not_write_raw_page_when_response_is_non_2xx(): void { Http::preventStrayRequests(); + Sleep::fake(); $listHtml = '
    • ' .'test' @@ -63,6 +68,7 @@ public function test_does_not_write_raw_page_when_response_is_non_2xx(): void new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false), new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)), new UpdateOrCreateRawPage, + new SleepBetweenRequests, ); $handler(new NullLogger); diff --git a/tests/Feature/Actions/Scrape/Japan/HandlerIsolationTest.php b/tests/Feature/Actions/Scrape/Japan/HandlerIsolationTest.php index a87c4b5..1b957ff 100644 --- a/tests/Feature/Actions/Scrape/Japan/HandlerIsolationTest.php +++ b/tests/Feature/Actions/Scrape/Japan/HandlerIsolationTest.php @@ -7,10 +7,12 @@ use App\Actions\Scrape\FetchHtml; use App\Actions\Scrape\Japan\FindUrls; use App\Actions\Scrape\Japan\Handler; +use App\Actions\Scrape\SleepBetweenRequests; use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Models\RawPage; use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Sleep; use Psr\Log\NullLogger; use Tests\Feature\TestCase; @@ -22,6 +24,7 @@ final class HandlerIsolationTest extends TestCase public function test_failure_on_one_url_does_not_stop_the_others(): void { Http::preventStrayRequests(); + Sleep::fake(); $listHtml = '
        ' .'
      • broken
        • ' @@ -38,6 +41,7 @@ public function test_failure_on_one_url_does_not_stop_the_others(): void new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false), new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)), new UpdateOrCreateRawPage, + new SleepBetweenRequests, ); $handler(new NullLogger); diff --git a/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php b/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php index afeff2a..9e801bb 100644 --- a/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php +++ b/tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php @@ -5,6 +5,7 @@ namespace Tests\Feature\Actions\Scrape\Twitrans; use App\Actions\Scrape\FetchHtml; +use App\Actions\Scrape\SleepBetweenRequests; use App\Actions\Scrape\Twitrans\FindUrls; use App\Actions\Scrape\Twitrans\Handler; use App\Actions\Scrape\UpdateOrCreateRawPage; @@ -35,6 +36,7 @@ public function test_backs_off_longer_after_429_than_after_success(): void new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false), new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)), new UpdateOrCreateRawPage, + new SleepBetweenRequests, ); $handler(new NullLogger);