diff --git a/app/Actions/Scrape/Japan/Handler.php b/app/Actions/Scrape/Japan/Handler.php index 875ea90..a14963c 100644 --- a/app/Actions/Scrape/Japan/Handler.php +++ b/app/Actions/Scrape/Japan/Handler.php @@ -6,18 +6,23 @@ use App\Actions\Scrape\FetchHtml; use App\Actions\Scrape\HandlerInterface; +use App\Actions\Scrape\SleepBetweenRequests; use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Enums\Encoding; use App\Enums\SiteName; -use Illuminate\Support\Sleep; use Psr\Log\LoggerInterface; final readonly class Handler implements HandlerInterface { + private const int INTERVAL_SECONDS = 2; + + private const int RATE_LIMIT_COOLDOWN_SECONDS = 15; + public function __construct( private FetchHtml $fetchHtml, private FindUrls $findUrls, private UpdateOrCreateRawPage $updateOrCreateRawPage, + private SleepBetweenRequests $sleepBetweenRequests, ) {} #[\Override] @@ -34,9 +39,10 @@ public function __invoke(LoggerInterface $logger): void SiteName::Japan, $html ); - Sleep::for(1)->second(); + ($this->sleepBetweenRequests)(null, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS); } catch (\Throwable $th) { $logger->error('failed', [$url, $th]); + ($this->sleepBetweenRequests)($th, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS); } } } diff --git a/app/Actions/Scrape/SleepBetweenRequests.php b/app/Actions/Scrape/SleepBetweenRequests.php new file mode 100644 index 0000000..7a22d50 --- /dev/null +++ b/app/Actions/Scrape/SleepBetweenRequests.php @@ -0,0 +1,23 @@ +response->status() === 429) { + Sleep::for($rateLimitCooldownSeconds)->seconds(); + + return; + } + + Sleep::for($intervalSeconds)->seconds(); + } +} diff --git a/app/Actions/Scrape/Twitrans/Handler.php b/app/Actions/Scrape/Twitrans/Handler.php index dc641cd..3c27b93 100644 --- a/app/Actions/Scrape/Twitrans/Handler.php +++ b/app/Actions/Scrape/Twitrans/Handler.php @@ -6,18 +6,23 @@ use App\Actions\Scrape\FetchHtml; use App\Actions\Scrape\HandlerInterface; +use App\Actions\Scrape\SleepBetweenRequests; use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Enums\Encoding; use App\Enums\SiteName; -use Illuminate\Support\Sleep; use Psr\Log\LoggerInterface; final readonly class Handler implements HandlerInterface { + private const int INTERVAL_SECONDS = 10; + + private const int RATE_LIMIT_COOLDOWN_SECONDS = 60; + public function __construct( private FetchHtml $fetchHtml, private FindUrls $findUrls, private UpdateOrCreateRawPage $updateOrCreateRawPage, + private SleepBetweenRequests $sleepBetweenRequests, ) {} #[\Override] @@ -34,9 +39,10 @@ public function __invoke(LoggerInterface $logger): void SiteName::Twitrans, $html ); - Sleep::for(1)->second(); + ($this->sleepBetweenRequests)(null, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS); } catch (\Throwable $th) { $logger->error('failed', [$url, $th]); + ($this->sleepBetweenRequests)($th, self::INTERVAL_SECONDS, self::RATE_LIMIT_COOLDOWN_SECONDS); } } } diff --git a/tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php b/tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php index 789c447..083fbc4 100644 --- a/tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php +++ b/tests/Feature/Actions/Scrape/Japan/HandlerFailureTest.php @@ -7,10 +7,12 @@ use App\Actions\Scrape\FetchHtml; use App\Actions\Scrape\Japan\FindUrls; use App\Actions\Scrape\Japan\Handler; +use App\Actions\Scrape\SleepBetweenRequests; use App\Actions\Scrape\UpdateOrCreateRawPage; use App\Models\RawPage; use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Sleep; use Psr\Log\NullLogger; use Tests\Feature\TestCase; @@ -23,6 +25,7 @@ final class HandlerFailureTest extends TestCase public function test_does_not_write_raw_page_when_fetch_fails(): void { Http::preventStrayRequests(); + Sleep::fake(); $listHtml = '