fix: Fixes broken scrape result issue for keywords with special chara…

…cters. closes #221
towfiqi · Nov 12, 2024 · d9d7c63 · d9d7c63
1 parent c5714c0
commit d9d7c63
Show file tree

Hide file tree

Showing 7 changed files with 14 additions and 14 deletions.
diff --git a/scrapers/services/hasdata.ts b/scrapers/services/hasdata.ts
@@ -20,11 +20,10 @@ const hasdata:ScraperSettings = {
    scrapeURL: (keyword, settings) => {
       const country = keyword.country || 'US';
       const countryName = countries[country][0];
-      const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
-      return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURI(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
+      const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
+      return `https://api.scrape-it.cloud/scrape/google/serp?q=${encodeURIComponent(keyword.keyword)}${location}&num=100&gl=${country.toLowerCase()}&deviceType=${keyword.device}`;
    },
    resultObjectKey: 'organicResults',
-
    serpExtractor: (content) => {
       const extractedResult = [];
       const results: HasDataResult[] = (typeof content === 'string') ? JSON.parse(content) : content as HasDataResult[];

diff --git a/scrapers/services/searchapi.ts b/scrapers/services/searchapi.ts
@@ -20,8 +20,8 @@ const searchapi:ScraperSettings = {
   scrapeURL: (keyword) => {
    const country = keyword.country || 'US';
    const countryName = countries[country][0];
-   const location = keyword.city && countryName ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
-     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURI(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
+   const location = keyword.city && countryName ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
+     return `https://www.searchapi.io/api/v1/search?engine=google&q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${country}&device=${keyword.device}${location}`;
   },
   resultObjectKey: 'organic_results',
   serpExtractor: (content) => {

diff --git a/scrapers/services/serpapi.ts b/scrapers/services/serpapi.ts
@@ -19,8 +19,8 @@ const serpapi:ScraperSettings = {
    },
    scrapeURL: (keyword, settings) => {
       const countryName = countries[keyword.country || 'US'][0];
-      const location = keyword.city && keyword.country ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
-      return `https://serpapi.com/search?q=${encodeURI(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
+      const location = keyword.city && keyword.country ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
+      return `https://serpapi.com/search?q=${encodeURIComponent(keyword.keyword)}&num=100&gl=${keyword.country}&device=${keyword.device}${location}&api_key=${settings.scaping_api}`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

diff --git a/scrapers/services/serper.ts b/scrapers/services/serper.ts
@@ -12,7 +12,8 @@ const serper:ScraperSettings = {
    scrapeURL: (keyword, settings, countryData) => {
       const country = keyword.country || 'US';
       const lang = countryData[country][2];
-      return `https://google.serper.dev/search?q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
+      console.log('Serper URL :', `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`);
+      return `https://google.serper.dev/search?q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}&num=100&apiKey=${settings.scaping_api}`;
    },
    resultObjectKey: 'organic',
    serpExtractor: (content) => {

diff --git a/scrapers/services/serply.ts b/scrapers/services/serply.ts
@@ -20,7 +20,7 @@ const serply:ScraperSettings = {
    },
    scrapeURL: (keyword) => {
       const country = scraperCountries.includes(keyword.country.toUpperCase()) ? keyword.country : 'US';
-      return `https://api.serply.io/v1/search/q=${encodeURI(keyword.keyword)}&num=100&hl=${country}`;
+      return `https://api.serply.io/v1/search/q=${encodeURIComponent(keyword.keyword)}&num=100&hl=${country}`;
    },
    resultObjectKey: 'result',
    serpExtractor: (content) => {

diff --git a/scrapers/services/spaceserp.ts b/scrapers/services/spaceserp.ts
@@ -15,10 +15,10 @@ const spaceSerp:ScraperSettings = {
    scrapeURL: (keyword, settings, countryData) => {
       const country = keyword.country || 'US';
       const countryName = countries[country][0];
-      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
       const device = keyword.device === 'mobile' ? '&device=mobile' : '';
       const lang = countryData[country][2];
-      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
+      return `https://api.spaceserp.com/google/search?apiKey=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&pageSize=100&gl=${country}&hl=${lang}${location}${device}&resultBlocks=`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {

diff --git a/scrapers/services/valueserp.ts b/scrapers/services/valueserp.ts
@@ -15,11 +15,11 @@ const valueSerp:ScraperSettings = {
    scrapeURL: (keyword, settings, countryData) => {
       const country = keyword.country || 'US';
       const countryName = countries[country][0];
-      const location = keyword.city ? `&location=${encodeURI(`${keyword.city},${countryName}`)}` : '';
+      const location = keyword.city ? `&location=${encodeURIComponent(`${keyword.city},${countryName}`)}` : '';
       const device = keyword.device === 'mobile' ? '&device=mobile' : '';
       const lang = countryData[country][2];
-      console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
-      return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURI(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
+      console.log(`https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`);
+      return `https://api.valueserp.com/search?api_key=${settings.scaping_api}&q=${encodeURIComponent(keyword.keyword)}&gl=${country}&hl=${lang}${device}${location}&num=100&output=json&include_answer_box=false&include_advertiser_info=false`;
    },
    resultObjectKey: 'organic_results',
    serpExtractor: (content) => {