Allow for multiple instances of web scraping sessions

kkamara · Mar 16, 2024 · 83ee136 · 83ee136
1 parent 0604f85
commit 83ee136
Show file tree

Hide file tree

Showing 5 changed files with 19 additions and 85 deletions.
diff --git a/.env.example b/.env.example
@@ -5,6 +5,8 @@ APP_DEBUG=true
 APP_TIMEZONE=UTC
 APP_URL=http://localhost
 
+SELENIUM_GRID_PORT=4444
+
 APP_LOCALE=en
 APP_FALLBACK_LOCALE=en
 APP_FAKER_LOCALE=en_US

diff --git a/app/Console/Commands/BrowserScrape.php b/app/Console/Commands/BrowserScrape.php
@@ -24,17 +24,14 @@ class BrowserScrape extends Command
     /**
      * @var Client
      */
-    protected Client $client;
+    private Client $client;
 
     public function __construct()
     {
         parent::__construct();
-        $this->client = Client::createChromeClient(null, [
-            '--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36',
-            '--window-size=1200,1100',
-            // '--headless',
-            // '--disable-gpu',
-        ]);
+        $this->client = Client::createSeleniumClient(
+            'http://localhost:'.config('app.selenium_grid_port').'/wd/hub'
+        );
     }
 
     /**

diff --git a/app/Console/Commands/TestCrawler.php b/app/Console/Commands/TestCrawler.php
diff --git a/config/app.php b/config/app.php
@@ -15,6 +15,8 @@
 
     'name' => env('APP_NAME', 'Laravel'),
 
+    'selenium_grid_port' => env('SELENIUM_GRID_PORT', 4444),
+
     /*
     |--------------------------------------------------------------------------
     | Application Environment

diff --git a/readme.md b/readme.md
@@ -45,6 +45,11 @@ cp .env.example .env
 composer install
 # install chromedriver for Panther client.
 vendor/bin/bdi detect drivers
+```
+
+#### Add ./drivers/ to your environment Path.
+
+```bash
 # Run composer install again.
 composer install
 php artisan key:generate
@@ -55,19 +60,14 @@ npm install
 npm run build
 ```
 
-#### The following installation step may or may not be required.
+#### Download Selenium Server jar file
+
+[Download Selenium Server jar file](https://www.selenium.dev/documentation/grid/getting_started/).
 
-[Installing web drivers](https://symfony.com/doc/current/testing/end_to_end.html#installing-web-drivers).
+Run the following in a new terminal.
 
 ```bash
-# chromedriver_mac64
-# chromedriver_win32
-# See https://chromedriver.storage.googleapis.com
-# for drivers list.
-wget https://chromedriver.storage.googleapis.com/2.37/chromedriver_linux64.zip
-unzip chromedriver_linux64.zip
-sudo mv chromedriver /usr/bin/chromedriver
-chromedriver --version
+java -jar selenium-server-4.18.1.jar standalone
 ```
 
 ## Usage
@@ -91,7 +91,7 @@ export PANTHER_DEVTOOLS='' # enabled
 ## Adding a new command <a name="adding-commands"></a>
 
 ```bash
-php artisan make:crawler crawler_test
+php artisan make:crawler TestCrawler
 ```
 
 ## Misc