Fetch Jobs

app/Http/Controllers/HouseDogController.php

@@ -0,0 +1,10 @@
+<?php
+
+namespace App\Http\Controllers;
+
+use Illuminate\Http\Request;
+
+class HouseDogController extends Controller
+{
+    //
+}

app/Jobs/GoFetchDogJob.php

@@ -0,0 +1,57 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\HouseDog;
+use App\Services\PantherService;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
+
+class GoFetchDogJob implements ShouldQueue, ShouldBeUnique
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    public $petId;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct($petId)
+    {
+        $this->petId = $petId;
+    }
+
+    public function uniqueId() {
+        return $this->petId;
+    }
+
+    /**
+     * @throws TransportExceptionInterface
+     * @throws ServerExceptionInterface
+     * @throws RedirectionExceptionInterface
+     * @throws ClientExceptionInterface
+     */
+    public function handle(PantherService $pantherService): void
+    {
+        $data = $pantherService->fetchData(config('services.panther.uris.card') . $this->petId .
+            config('services.panther.uris.cardSuffix'));
+        HouseDog::updateOrCreate(
+            ['id' => $this->petId],
+            [
+                'photoUri' => $data['photoUrl'], // Comes from DD, do not change
+                'size' => $data['size']
+            ]
+        );
+
+        sleep(4);
+    }
+
+}

app/Jobs/GoFetchListJob.php

@@ -0,0 +1,72 @@
+<?php
+
+namespace App\Jobs;
+
+use App\Models\HouseDog;
+use App\Services\PantherService;
+use Illuminate\Bus\Queueable;
+use Illuminate\Contracts\Queue\ShouldBeUnique;
+use Illuminate\Contracts\Queue\ShouldQueue;
+use Illuminate\Foundation\Bus\Dispatchable;
+use Illuminate\Queue\InteractsWithQueue;
+use Illuminate\Queue\SerializesModels;
+use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
+
+class GoFetchListJob implements ShouldQueue, ShouldBeUnique
+{
+    use Dispatchable, InteractsWithQueue, Queueable, SerializesModels;
+
+    /**
+     * Create a new job instance.
+     */
+    public function __construct()
+    {
+        $this->onQueue('high');
+    }
+
+    /**
+     * @throws TransportExceptionInterface
+     * @throws ServerExceptionInterface
+     * @throws RedirectionExceptionInterface
+     * @throws ClientExceptionInterface
+     */
+    public function handle(PantherService $pantherService): void
+    {
+        $start = hrtime(true);
+
+        $data = $pantherService->fetchData(config('services.panther.uris.inHouseList'));
+        $columns = collect($data['columns'])->pluck('index', 'filterKey');
+
+
+        $existingIds = HouseDog::pluck('id')->toArray();
+        $newIds = collect($data['rows'])->pluck('id')->toArray();
+        $idsToDelete = array_diff($existingIds, $newIds);
+        HouseDog::whereIn('id', $idsToDelete)->delete();
+
+        foreach ($data['rows'] as $row) {
+            $petId = $row[$columns['petId']];
+            HouseDog::updateOrCreate(
+                ['id' => $petId],
+                [
+                    'name' => $this->trimToNull($row[$columns['name']]),
+                    'gender' => $this->trimToNull($row[$columns['gender']]),
+                    'cabinName' => $this->trimToNull($row[$columns['dateCabin']])
+                ]
+            );
+            GoFetchDogJob::dispatch($petId, $pantherService);
+        }
+
+        $stop = hrtime(true);
+        sleep(4 - (($stop - $start) / 1e9));
+    }
+
+    function trimToNull($value) {
+        $trimmed = trim($value);
+        return $trimmed === '' ? null : $trimmed;
+    }
+
+
+}

app/Models/HouseDog.php

@@ -0,0 +1,12 @@
+<?php
+
+namespace App\Models;
+
+use Illuminate\Database\Eloquent\Factories\HasFactory;
+use Illuminate\Database\Eloquent\Model;
+
+class HouseDog extends Model
+{
+    use HasFactory;
+    protected $fillable = ['id', 'name', 'gender', 'photoUri', 'size', 'cabinName'];
+}

app/Services/PantherService.php

@@ -0,0 +1,134 @@
+<?php
+
+namespace App\Services;
+
+use Carbon\Carbon;
+use Illuminate\Support\Facades\Cache;
+use Illuminate\Support\Facades\Log;
+use Symfony\Component\BrowserKit\Cookie;
+use Symfony\Component\HttpClient\HttpClient;
+use Symfony\Component\Panther\Client;
+use Symfony\Component\Panther\Cookie\CookieJar;
+use Symfony\Component\Panther\DomCrawler\Crawler;
+use Symfony\Contracts\HttpClient\Exception\ClientExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\RedirectionExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\ServerExceptionInterface;
+use Symfony\Contracts\HttpClient\Exception\TransportExceptionInterface;
+
+class PantherService
+{
+    protected Client $client;
+    const SUBMIT_BUTTON_CSS = 'button[type=submit]'; // Login
+
+    public function __construct()
+    {
+        $this->client = Client::createChromeClient(null, null, [
+            'port' => intval(config('services.panther.port')),
+        ]);
+    }
+
+    public function loginAndStoreCookie(): void
+    {
+        try {
+            $this->client->request('GET', config('services.panther.uris.base'));
+            $crawler = $this->client->waitFor(self::SUBMIT_BUTTON_CSS);
+
+            $form = $crawler->selectButton('Submit')->form();
+            $form['username'] = config('services.panther.username');
+            $form['password'] = config('services.panther.password');
+            $this->client->submit($form);
+
+            $element = $this->waitForEitherElement(['.cbw-messaging-error', 'div#in-house']);
+            if (str_contains($element->attr('class'), 'cbw-messaging-error')) {
+                Log::error("Form submission failed with error: " . $element->text());
+            }
+            $this->storeCookieJar($this->client->getCookieJar());
+
+        } catch (\Exception $e) {
+            Log::error('Error requesting base_uri:' . $e->getMessage());
+        } finally {
+            $this->client->quit();
+        }
+    }
+
+    /**
+     * @throws TransportExceptionInterface
+     * @throws ServerExceptionInterface
+     * @throws RedirectionExceptionInterface
+     * @throws ClientExceptionInterface
+     */
+    public function fetchData(string $url): ?array
+    {
+        try {
+            $cookies = $this->getExistingCookieJar();
+            if (!$cookies) {
+
+                $this->loginAndStoreCookie();
+                $cookies = $this->getExistingCookieJar();
+            }
+            $httpClient = HttpClient::create();
+            $cookieArray = [];
+            if (count($cookies) > 0) {
+                foreach ($cookies as $cookie) {
+                    $cookieArray[] = $cookie['name'] . '=' . $cookie['value'];
+                }
+            }
+
+            $response = $httpClient->request('GET', $url, [
+                'headers' => [
+                    'Cookie' => implode('; ', $cookieArray),
+                ],
+            ]);
+            $data = json_decode($response->getContent(), true)['data'][0];
+            if (json_last_error() === JSON_ERROR_NONE) {
+                return $data;
+            } else {
+                Log::error('Failed to parse JSON: ' . json_last_error_msg());
+            }
+        } catch (\Exception $e) {
+            Log::error('Error requesting ' . $url . ': ' . $e->getMessage());
+            dd($e);
+        }
+        return null;
+    }
+
+    protected function getExistingCookieJar(): ?array
+    {
+        return Cache::get('panther_cookie_jar');
+    }
+
+    protected function storeCookieJar(CookieJar $cookieJar): void
+    {
+        $expiration = now()->addHour();
+        $cookies = [];
+        foreach ($cookieJar->all() as $cookie) {
+            if ($cookie->getExpiresTime() !== null) {
+                $expiration = Carbon::createFromTimestamp($cookie->getExpiresTime());
+            }
+            $cookies[] = ['name' => $cookie->getName(), 'value' => $cookie->getValue()];
+        }
+        Cache::put('panther_cookie_jar', $cookies, $expiration);
+    }
+
+
+    protected function waitForEitherElement(array $selectors, int $timeout = 15): ?Crawler
+    {
+        $endTime = time() + $timeout;
+
+        while (time() < $endTime) {
+            foreach ($selectors as $selector) {
+                try {
+                    $element = $this->client->getCrawler()->filter($selector);
+                    if ($element->count() > 0) {
+                        return $element->first();
+                    }
+                } catch (\InvalidArgumentException $e) {
+                    continue;
+                }
+            }
+            usleep(500000); // 500 milliseconds
+        }
+
+        throw new \RuntimeException('None of the elements were found within the timeout period.');
+    }
+}

composer.json

@@ -6,18 +6,21 @@
     "license": "MIT",
     "require": {
         "php": "^8.2",
+        "ext-dom": "*",
         "inertiajs/inertia-laravel": "^1.0",
         "laravel/framework": "^11.9",
         "laravel/sanctum": "^4.0",
         "laravel/telescope": "^5.1",
         "laravel/tinker": "^2.9",
+        "symfony/panther": "^2.1",
         "tightenco/ziggy": "^2.0"
     },
     "require-dev": {
+        "dbrekelmans/bdi": "^1.3",
         "fakerphp/faker": "^1.23",
         "laravel/breeze": "^2.1",
+        "laravel/dusk": "^8.2",
         "laravel/pint": "^1.13",
-        "laravel/sail": "^1.26",
         "mockery/mockery": "^1.6",
         "nunomaduro/collision": "^8.0",
         "phpunit/phpunit": "^11.0.1"