fix(scrape): sleep after failure too, back off longer on 429

app/Actions/Scrape/Japan/Handler.php

@@ -6,18 +6,23 @@
 
 use App\Actions\Scrape\FetchHtml;
 use App\Actions\Scrape\HandlerInterface;
+use App\Actions\Scrape\SleepBetweenRequests;
 use App\Actions\Scrape\UpdateOrCreateRawPage;
 use App\Enums\Encoding;
 use App\Enums\SiteName;
-use Illuminate\Support\Sleep;
 use Psr\Log\LoggerInterface;
 
 final readonly class Handler implements HandlerInterface
 {
+    private const int INTERVAL_SECONDS = 2;
+
+    private const int RATE_LIMIT_COOLDOWN_SECONDS = 15;
+
     public function __construct(
         private FetchHtml $fetchHtml,
         private FindUrls $findUrls,
         private UpdateOrCreateRawPage $updateOrCreateRawPage,
+        private SleepBetweenRequests $sleepBetweenRequests,
     ) {}
 
     #[\Override]
@@ -34,9 +39,10 @@ public function __invoke(LoggerInterface $logger): void
                     SiteName::Japan,
                     $html
                 );
-                Sleep::for(1)->second();
+                ($this->sleepBetweenRequests)(null, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS);
             } catch (\Throwable $th) {
                 $logger->error('failed', [$url, $th]);
+                ($this->sleepBetweenRequests)($th, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS);
             }
         }
     }

app/Actions/Scrape/SleepBetweenRequests.php

@@ -0,0 +1,23 @@
+<?php
+
+declare(strict_types=1);
+
+namespace App\Actions\Scrape;
+
+use Illuminate\Http\Client\RequestException;
+use Illuminate\Support\Sleep;
+
+final class SleepBetweenRequests
+{
+    public function __invoke(?\Throwable $throwable, int $intervalSeconds, int $rateLimitCooldownSeconds): void
+    {
+        // 429 は通常のリトライ間隔では解消しないため、長めに待って次の URL へ進む。
+        if ($throwable instanceof RequestException && $throwable->response->status() === 429) {
+            Sleep::for($rateLimitCooldownSeconds)->seconds();
+
+            return;
+        }
+
+        Sleep::for($intervalSeconds)->seconds();
+    }
+}

app/Actions/Scrape/Twitrans/Handler.php

@@ -6,18 +6,23 @@
 
 use App\Actions\Scrape\FetchHtml;
 use App\Actions\Scrape\HandlerInterface;
+use App\Actions\Scrape\SleepBetweenRequests;
 use App\Actions\Scrape\UpdateOrCreateRawPage;
 use App\Enums\Encoding;
 use App\Enums\SiteName;
-use Illuminate\Support\Sleep;
 use Psr\Log\LoggerInterface;
 
 final readonly class Handler implements HandlerInterface
 {
+    private const int INTERVAL_SECONDS = 10;
+
+    private const int RATE_LIMIT_COOLDOWN_SECONDS = 60;
+
     public function __construct(
         private FetchHtml $fetchHtml,
         private FindUrls $findUrls,
         private UpdateOrCreateRawPage $updateOrCreateRawPage,
+        private SleepBetweenRequests $sleepBetweenRequests,
     ) {}
 
     #[\Override]
@@ -34,9 +39,10 @@ public function __invoke(LoggerInterface $logger): void
                     SiteName::Twitrans,
                     $html
                 );
-                Sleep::for(1)->second();
+                ($this->sleepBetweenRequests)(null, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS);
             } catch (\Throwable $th) {
                 $logger->error('failed', [$url, $th]);
+                ($this->sleepBetweenRequests)($th, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS);
             }
         }
     }

tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php

@@ -7,10 +7,12 @@
 use App\Actions\Scrape\FetchHtml;
 use App\Actions\Scrape\Japan\FindUrls;
 use App\Actions\Scrape\Japan\Handler;
+use App\Actions\Scrape\SleepBetweenRequests;
 use App\Actions\Scrape\UpdateOrCreateRawPage;
 use App\Models\RawPage;
 use Illuminate\Http\Client\ConnectionException;
 use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Sleep;
 use Psr\Log\NullLogger;
 use Tests\Feature\TestCase;
 
@@ -23,6 +25,7 @@ final class HandlerFailureTest extends TestCase
     public function test_does_not_write_raw_page_when_fetch_fails(): void
     {
         Http::preventStrayRequests();
+        Sleep::fake();
 
         $listHtml = '<html><body><div id="body"><ul><li>'
             .'<a href="https://japanese.simutrans.com:443/index.php?Addon128%2FTest">test</a>'
@@ -39,6 +42,7 @@ public function test_does_not_write_raw_page_when_fetch_fails(): void
             new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false),
             new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)),
             new UpdateOrCreateRawPage,
+            new SleepBetweenRequests,
         );
 
         $handler(new NullLogger);
@@ -49,6 +53,7 @@ public function test_does_not_write_raw_page_when_fetch_fails(): void
     public function test_does_not_write_raw_page_when_response_is_non_2xx(): void
     {
         Http::preventStrayRequests();
+        Sleep::fake();
 
         $listHtml = '<html><body><div id="body"><ul><li>'
             .'<a href="https://japanese.simutrans.com:443/index.php?Addon128%2FTest">test</a>'
@@ -63,6 +68,7 @@ public function test_does_not_write_raw_page_when_response_is_non_2xx(): void
             new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false),
             new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)),
             new UpdateOrCreateRawPage,
+            new SleepBetweenRequests,
         );
 
         $handler(new NullLogger);

tests/Feature/Actions/Scrape/Japan/HandlerIsolationTest.php

@@ -7,10 +7,12 @@
 use App\Actions\Scrape\FetchHtml;
 use App\Actions\Scrape\Japan\FindUrls;
 use App\Actions\Scrape\Japan\Handler;
+use App\Actions\Scrape\SleepBetweenRequests;
 use App\Actions\Scrape\UpdateOrCreateRawPage;
 use App\Models\RawPage;
 use Illuminate\Http\Client\ConnectionException;
 use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Sleep;
 use Psr\Log\NullLogger;
 use Tests\Feature\TestCase;
 
@@ -22,6 +24,7 @@ final class HandlerIsolationTest extends TestCase
     public function test_failure_on_one_url_does_not_stop_the_others(): void
     {
         Http::preventStrayRequests();
+        Sleep::fake();
 
         $listHtml = '<html><body><div id="body"><ul>'
             .'<li><a href="https://japanese.simutrans.com:443/index.php?Addon128%2FBroken">broken</a></li>'
@@ -38,6 +41,7 @@ public function test_failure_on_one_url_does_not_stop_the_others(): void
             new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false),
             new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)),
             new UpdateOrCreateRawPage,
+            new SleepBetweenRequests,
         );
 
         $handler(new NullLogger);

tests/Feature/Actions/Scrape/Twitrans/HandlerRateLimitTest.php

@@ -0,0 +1,50 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Tests\Feature\Actions\Scrape\Twitrans;
+
+use App\Actions\Scrape\FetchHtml;
+use App\Actions\Scrape\SleepBetweenRequests;
+use App\Actions\Scrape\Twitrans\FindUrls;
+use App\Actions\Scrape\Twitrans\Handler;
+use App\Actions\Scrape\UpdateOrCreateRawPage;
+use Illuminate\Support\Facades\Http;
+use Illuminate\Support\Sleep;
+use Psr\Log\NullLogger;
+use Tests\Feature\TestCase;
+
+final class HandlerRateLimitTest extends TestCase
+{
+    public function test_backs_off_longer_after_429_than_after_success(): void
+    {
+        Http::preventStrayRequests();
+        Sleep::fake();
+
+        $listHtml = '<html><body><div id="content"><ul>'
+            .'<li><a href="/twitrans/addon/pak128.japan/Ok">ok</a></li>'
+            .'<li><a href="/twitrans/addon/pak128.japan/Limited">limited</a></li>'
+            .'</ul></div></body></html>';
+
+        Http::fake([
+            'https://wikiwiki.jp/twitrans?cmd=list' => Http::response($listHtml, 200),
+            'https://wikiwiki.jp/twitrans/addon/pak128.japan/Ok' => Http::response('<html><body>ok</body></html>', 200),
+            'https://wikiwiki.jp/twitrans/addon/pak128.japan/Limited' => Http::response('rate limited', 429),
+        ]);
+
+        $handler = new Handler(
+            new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false),
+            new FindUrls(new FetchHtml(retryTimes: 1, sleepMilliseconds: 1, useCache: false)),
+            new UpdateOrCreateRawPage,
+            new SleepBetweenRequests,
+        );
+
+        $handler(new NullLogger);
+
+        // 成功時は通常間隔(10秒)、429 を受けた直後はより長いクールダウン(60秒)。
+        Sleep::assertSequence([
+            Sleep::for(10)->seconds(),
+            Sleep::for(60)->seconds(),
+        ]);
+    }
+}