From 49b4769528d18e34c16386b73dfb662e7a9f45a0 Mon Sep 17 00:00:00 2001 From: Towfiq Date: Tue, 20 Dec 2022 13:24:29 +0600 Subject: [PATCH] feat: integrates Google Search Console. --- README.md | 3 +- components/common/Chart.tsx | 9 +- components/common/Icon.tsx | 59 ++++ components/common/SelectField.tsx | 4 +- components/common/Sidebar.tsx | 7 +- components/common/TopBar.tsx | 15 +- components/domains/DomainHeader.tsx | 110 ++++-- components/domains/DomainItem.tsx | 75 ++++ components/domains/DomainSettings.tsx | 10 +- components/insight/Insight.tsx | 131 +++++++ components/insight/InsightItem.tsx | 69 ++++ components/insight/InsightStats.tsx | 123 +++++++ components/keywords/AddKeywords.tsx | 4 +- components/keywords/Keyword.tsx | 48 ++- components/keywords/KeywordDetails.tsx | 2 +- components/keywords/KeywordFilter.tsx | 61 +++- components/keywords/KeywordsTable.tsx | 67 +++- components/keywords/SCKeyword.tsx | 75 ++++ components/keywords/SCKeywordsTable.tsx | 227 ++++++++++++ cron.js | 15 + email/email.html | 26 +- package-lock.json | 436 +++++++++++++++++++++++- package.json | 4 + pages/_app.tsx | 8 +- pages/api/domains.ts | 8 +- pages/api/insight.ts | 54 +++ pages/api/keywords.ts | 22 +- pages/api/notify.ts | 76 +++-- pages/api/searchconsole.ts | 65 ++++ pages/api/settings.ts | 7 +- pages/domain/[slug]/index.tsx | 17 +- pages/domain/console/[slug]/index.tsx | 89 +++++ pages/domain/insight/[slug]/index.tsx | 89 +++++ pages/domains/index.tsx | 91 +++++ pages/index.tsx | 28 +- services/domains.tsx | 14 +- services/keywords.tsx | 12 +- services/searchConsole.ts | 38 +++ styles/globals.css | 91 ++++- types.d.ts | 106 +++++- utils/SCsortFilter.ts | 82 +++++ utils/countries.ts | 256 ++++++++++++++ utils/domains.ts | 45 +++ utils/exportcsv.ts | 31 +- utils/generateEmail.ts | 130 ++++++- utils/insight.ts | 142 ++++++++ utils/searchConsole.ts | 154 +++++++++ utils/sortFilter.ts | 40 ++- utils/verifyUser.ts | 12 +- 49 files changed, 3081 insertions(+), 206 deletions(-) create mode 100644 components/domains/DomainItem.tsx create mode 100644 components/insight/Insight.tsx create mode 100644 components/insight/InsightItem.tsx create mode 100644 components/insight/InsightStats.tsx create mode 100644 components/keywords/SCKeyword.tsx create mode 100644 components/keywords/SCKeywordsTable.tsx create mode 100644 pages/api/insight.ts create mode 100644 pages/api/searchconsole.ts create mode 100644 pages/domain/console/[slug]/index.tsx create mode 100644 pages/domain/insight/[slug]/index.tsx create mode 100644 pages/domains/index.tsx create mode 100644 services/searchConsole.ts create mode 100644 utils/SCsortFilter.ts create mode 100644 utils/domains.ts create mode 100644 utils/insight.ts create mode 100644 utils/searchConsole.ts diff --git a/README.md b/README.md index 39d2ee7..e203240 100644 --- a/README.md +++ b/README.md @@ -13,11 +13,12 @@ SerpBear is an Open Source Search Engine Position Tracking App. It allows you to - **Unlimited Keywords:** Add unlimited domains and unlimited keywords to track their SERP. - **Email Notification:** Get notified of your keyword position changes daily/weekly/monthly through email. - **SERP API:** SerpBear comes with built-in API that you can use for your marketing & data reporting tools. +- **Google Search Console Integration:** Get the actual visit count, impressions & more for Each keyword. - **Mobile App:** Add the PWA app to your mobile for a better mobile experience. - **Zero Cost to RUN:** Run the App on mogenius.com or Fly.io for free. #### How it Works -The App uses third party website scrapers like ScrapingAnt, ScrapingRobot or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. +The App uses third party website scrapers like ScrapingAnt, ScrapingRobot or Your given Proxy ips to scrape google search results to see if your domain appears in the search result for the given keyword. Also, When you connect your Googel Search Console account, the app shows actual search visits for each tracked keywords. You can also discover new keywords, and find the most performing keywords, countries, pages. #### Getting Started - **Step 1:** Deploy & Run the App. diff --git a/components/common/Chart.tsx b/components/common/Chart.tsx index 01d7ffe..d5ff0e4 100644 --- a/components/common/Chart.tsx +++ b/components/common/Chart.tsx @@ -6,19 +6,20 @@ ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, T type ChartProps ={ labels: string[], - sreies: number[] + sreies: number[], + reverse? : boolean, } -const Chart = ({ labels, sreies }:ChartProps) => { +const Chart = ({ labels, sreies, reverse = true }:ChartProps) => { const options = { responsive: true, maintainAspectRatio: false, animation: false as const, scales: { y: { - reverse: true, + reverse, min: 1, - max: 100, + max: reverse ? 100 : undefined, }, }, plugins: { diff --git a/components/common/Icon.tsx b/components/common/Icon.tsx index 7e054bb..03d4b52 100644 --- a/components/common/Icon.tsx +++ b/components/common/Icon.tsx @@ -186,6 +186,65 @@ const Icon = ({ type, color = 'currentColor', size = 16, title = '', classes = ' } + {type === 'idea' + && + + + + + + + } + {type === 'tracking' + && + + + } + {type === 'google' + && + + + + + + } + {type === 'cursor' + && + + + } + {type === 'eye' + && + + + + + + } + {type === 'target' + && + + + } + {type === 'help' + && + + + } + {type === 'date' + && + + + + + + + + + + + + } ); }; diff --git a/components/common/SelectField.tsx b/components/common/SelectField.tsx index 384ec44..aa6cbe4 100644 --- a/components/common/SelectField.tsx +++ b/components/common/SelectField.tsx @@ -71,7 +71,7 @@ const SelectField = (props: SelectFieldProps) => { className={`selected flex border ${rounded} p-1.5 px-4 cursor-pointer select-none w-[180px] min-w-[${minWidth}px] ${showOptions ? 'border-indigo-200' : ''}`} onClick={() => setShowOptions(!showOptions)}> - + {selected.length > 0 ? (selectedLabels.slice(0, 2).join(', ')) : defaultLabel} {multiple && selected.length > 2 @@ -99,7 +99,7 @@ const SelectField = (props: SelectFieldProps) => { return (
  • selectItem(opt)} > diff --git a/components/common/Sidebar.tsx b/components/common/Sidebar.tsx index 7cc8733..d88840d 100644 --- a/components/common/Sidebar.tsx +++ b/components/common/Sidebar.tsx @@ -4,7 +4,7 @@ import { useRouter } from 'next/router'; import Icon from './Icon'; type SidebarProps = { - domains: Domain[], + domains: DomainType[], showAddModal: Function } @@ -23,8 +23,9 @@ const Sidebar = ({ domains, showAddModal } : SidebarProps) => { className={'my-2.5 leading-10'}> + rounded-r-none ${((`/domain/${d.slug}` === router.asPath || `/domain/console/${d.slug}` === router.asPath + || `/domain/insight/${d.slug}` === router.asPath) + ? 'bg-white text-zinc-800 border border-r-0' : 'text-zinc-500')}`}> {d.domain.charAt(0)} diff --git a/components/common/TopBar.tsx b/components/common/TopBar.tsx index 5e3447e..e0c2ac0 100644 --- a/components/common/TopBar.tsx +++ b/components/common/TopBar.tsx @@ -1,3 +1,4 @@ +import Link from 'next/link'; import { useRouter } from 'next/router'; import React, { useState } from 'react'; import toast from 'react-hot-toast'; @@ -11,6 +12,7 @@ type TopbarProps = { const TopBar = ({ showSettings, showAddModal }:TopbarProps) => { const [showMobileMenu, setShowMobileMenu] = useState(false); const router = useRouter(); + const isDomainsPage = router.pathname === '/domains'; const logoutUser = async () => { try { @@ -28,12 +30,21 @@ const TopBar = ({ showSettings, showAddModal }:TopbarProps) => { }; return ( -
    +
    -

    +

    SerpBear

    + {!isDomainsPage && ( + +     + + + + )}
    + } + {isInsight && }
    exportCsv()}> Export as csv - + {!isConsole && !isInsight && ( + + )} + Domain Settings +
    - + {!isConsole && !isInsight && ( + + )} + {isConsole && ( +
    + {/* Data From Last: */} + setShowSCDates(!ShowSCDates)}> + {daysName(scFilter)} + + {ShowSCDates && ( +
    + {['threeDays', 'sevenDays', 'thirtyDays'].map((itemKey) => { + return ; + })} +
    + )} +
    + )}
    +
    ); }; diff --git a/components/domains/DomainItem.tsx b/components/domains/DomainItem.tsx new file mode 100644 index 0000000..e4db635 --- /dev/null +++ b/components/domains/DomainItem.tsx @@ -0,0 +1,75 @@ +/* eslint-disable @next/next/no-img-element */ +// import { useRouter } from 'next/router'; +// import { useState } from 'react'; +import TimeAgo from 'react-timeago'; +import dayjs from 'dayjs'; +import Link from 'next/link'; +import Icon from '../common/Icon'; + +type DomainItemProps = { + domain: DomainType, + selected: boolean, + isConsoleIntegrated: boolean +} + +const DomainItem = ({ domain, selected, isConsoleIntegrated = false }: DomainItemProps) => { + const { keywordsUpdated, slug, keywordCount = 0, avgPosition = 0, scVisits = 0, scImpressions = 0, scPosition = 0 } = domain; + // const router = useRouter(); + return ( +
    + + +
    +
    + {domain.domain} +
    +
    +

    {domain.domain}

    + {keywordsUpdated && ( + + Updated + + )} +
    +
    +
    +
    + Tracker +
    +
    +
    + Keywords{keywordCount} +
    +
    + Avg position{avgPosition} +
    +
    +
    + {isConsoleIntegrated && ( +
    +
    + Search Console (7d) +
    +
    +
    + Visits + {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(scVisits).replace('T', 'K')} +
    +
    + Impressions + {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(scImpressions).replace('T', 'K')} +
    +
    + Avg position + {scPosition} +
    +
    +
    + )} +     + +
    + ); +}; + +export default DomainItem; diff --git a/components/domains/DomainSettings.tsx b/components/domains/DomainSettings.tsx index 90d4373..6b60968 100644 --- a/components/domains/DomainSettings.tsx +++ b/components/domains/DomainSettings.tsx @@ -5,8 +5,7 @@ import Modal from '../common/Modal'; import { useDeleteDomain, useUpdateDomain } from '../../services/domains'; type DomainSettingsProps = { - domain:Domain|false, - domains: Domain[], + domain:DomainType|false, closeModal: Function } @@ -15,7 +14,7 @@ type DomainSettingsError = { msg: string, } -const DomainSettings = ({ domain, domains, closeModal }: DomainSettingsProps) => { +const DomainSettings = ({ domain, closeModal }: DomainSettingsProps) => { const router = useRouter(); const [showRemoveDomain, setShowRemoveDomain] = useState(false); const [settingsError, setSettingsError] = useState({ type: '', msg: '' }); @@ -24,10 +23,7 @@ const DomainSettings = ({ domain, domains, closeModal }: DomainSettingsProps) => const { mutate: updateMutate } = useUpdateDomain(() => closeModal(false)); const { mutate: deleteMutate } = useDeleteDomain(() => { closeModal(false); - const fitleredDomains = domain && domains.filter((d:Domain) => d.domain !== domain.domain); - if (fitleredDomains && fitleredDomains[0] && fitleredDomains[0].slug) { - router.push(`/domain/${fitleredDomains[0].slug}`); - } + router.push('/domains'); }); useEffect(() => { diff --git a/components/insight/Insight.tsx b/components/insight/Insight.tsx new file mode 100644 index 0000000..fa3ca30 --- /dev/null +++ b/components/insight/Insight.tsx @@ -0,0 +1,131 @@ +import React, { useState } from 'react'; +import { Toaster } from 'react-hot-toast'; +import { sortInsightItems } from '../../utils/insight'; +import SelectField from '../common/SelectField'; +import InsightItem from './InsightItem'; +import InsightStats from './InsightStats'; + +type SCInsightProps = { + domain: DomainType | null, + insight: InsightDataType, + isLoading: boolean, + isConsoleIntegrated: boolean, +} + +const SCInsight = ({ insight, isLoading = true, isConsoleIntegrated = true }: SCInsightProps) => { + const [activeTab, setActiveTab] = useState('stats'); + + const insightItems = insight[activeTab as keyof InsightDataType]; + const startDate = insight && insight.stats && insight.stats.length > 0 ? new Date(insight.stats[0].date) : null; + const endDate = insight && insight.stats && insight.stats.length > 0 ? new Date(insight.stats[insight.stats.length - 1].date) : null; + + const switchTab = (tab: string) => { + // window.insightTab = tab; + setActiveTab(tab); + }; + + const renderTableHeader = () => { + const headerNames: {[key:string]: string[]} = { + stats: ['Date', 'Avg Position', 'Visits', 'Impressions', 'CTR'], + keywords: ['Keyword', 'Avg Position', 'Visits ↑', 'Impressions', 'CTR', 'Countries'], + countries: ['Country', 'Avg Position', 'Visits ↑', 'Impressions', 'CTR', 'Keywords'], + pages: ['Page', 'Avg Position', 'Visits ↑', 'Impressions', 'CTR', 'Countries', 'Keywords'], + }; + + return ( + + ); + }; + + const deviceTabStyle = 'select-none cursor-pointer px-3 py-2 rounded-3xl mr-2'; + const deviceTabCountStyle = 'px-2 py-0 rounded-3xl bg-[#DEE1FC] text-[0.7rem] font-bold ml-1'; + + return ( +
    +
    +
    +
    +
      + {['stats', 'keywords', 'countries', 'pages'].map((tabItem) => { + const tabInsightItem = insight[tabItem as keyof InsightDataType]; + return
    • switchTab(tabItem)}> + {tabItem} + {tabItem !== 'stats' && ( + + {tabInsightItem && tabInsightItem.length ? tabInsightItem.length : 0} + + )} +
      • ; + })} +
    +
    + { return { label: d, value: d }; })} + selected={[activeTab]} + defaultLabel="Select Tab" + updateField={(updatedTab:[string]) => switchTab(updatedTab[0])} + multiple={false} + rounded={'rounded'} + /> +
    +
    +
    + {startDate && new Intl.DateTimeFormat('en-US', { dateStyle: 'medium' }).format(new Date(startDate))} + - + {endDate && new Intl.DateTimeFormat('en-US', { dateStyle: 'medium' }).format(new Date(endDate))} + (Last 30 Days) +
    +
    + {activeTab === 'stats' && ( + + )} + +
    +
    + {renderTableHeader()} +
    + {['keywords', 'pages', 'countries', 'stats'].includes(activeTab) && insight && insightItems + && (activeTab === 'stats' ? [...insightItems].reverse() : sortInsightItems(insightItems)).map( + (item:SCInsightItem, index: number) => { + const insightItemCount = insight ? insightItems : []; + const lastItem = !!(insightItemCount && (index === insightItemCount.length)); + return ; + }, + ) + } + {isConsoleIntegrated && isLoading && ( +

    Loading Insight...

    + )} + {!isConsoleIntegrated && ( +

    + Goolge Search has not been Integrated yet. Please See the Docs to Learn how to integrate Google Search Data for this Domain. +

    + )} +
    +
    +
    +
    + +
    + ); + }; + + export default SCInsight; diff --git a/components/insight/InsightItem.tsx b/components/insight/InsightItem.tsx new file mode 100644 index 0000000..22e729d --- /dev/null +++ b/components/insight/InsightItem.tsx @@ -0,0 +1,69 @@ +import React from 'react'; +import countries from '../../utils/countries'; +import Icon from '../common/Icon'; + +type InsightItemProps = { + item: SCInsightItem, + lastItem: boolean, + type: string +} + +const InsightItem = ({ item, lastItem, type }:InsightItemProps) => { + const { clicks, impressions, ctr, position, country = 'zzz', keyword, page, keywords = 0, countries: cntrs = 0, date } = item; + let firstItem = keyword; + if (type === 'pages') { firstItem = page; } if (type === 'stats') { + firstItem = date && new Intl.DateTimeFormat('en-US', { dateStyle: 'medium' }).format(new Date(date)); + } + if (type === 'countries') { firstItem = countries[country] && countries[country][0]; } + const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num); + + return ( +
    + +
    + {type === 'countries' && } + {firstItem} +
    + +
    + + + + {Math.round(position)} +
    + + {/*
    {formattedNum(clicks)}
    */} +
    + {formattedNum(clicks)} + Visits +
    + +
    + + + + {formattedNum(impressions)} +
    + +
    + + + + {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(ctr)}% +
    + + {(type === 'pages' || type === 'keywords') && ( +
    {formattedNum(cntrs)}
    + )} + + {(type === 'countries' || type === 'pages') && ( +
    {formattedNum(keywords)}
    + )} +
    + ); +}; + +export default InsightItem; diff --git a/components/insight/InsightStats.tsx b/components/insight/InsightStats.tsx new file mode 100644 index 0000000..34ae00b --- /dev/null +++ b/components/insight/InsightStats.tsx @@ -0,0 +1,123 @@ +import React, { useMemo, useState, useEffect } from 'react'; +import { Chart as ChartJS, CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend } from 'chart.js'; +import { Line } from 'react-chartjs-2'; + +ChartJS.register(CategoryScale, LinearScale, PointElement, LineElement, Title, Tooltip, Legend); + +type InsightStatsProps = { + stats: SearchAnalyticsStat[], + totalKeywords: number, + totalCountries: number, + totalPages: number, +} + +const InsightStats = ({ stats = [], totalKeywords = 0, totalPages = 0 }:InsightStatsProps) => { + const formattedNum = (num:number) => new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(num); + const [totalStat, setTotalStat] = useState({ impressions: 0, clicks: 0, ctr: 0, position: 0 }); + + useEffect(() => { + if (stats.length > 0) { + const totalStats = stats.reduce((acc, item) => { + return { + impressions: item.impressions + acc.impressions, + clicks: item.clicks + acc.clicks, + ctr: item.ctr + acc.ctr, + position: item.position + acc.position, + }; + }, { impressions: 0, clicks: 0, ctr: 0, position: 0 }); + setTotalStat(totalStats); + } + }, [stats]); + + const chartData = useMemo(() => { + const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + const chartSeries: {[key:string]: number[]} = { clicks: [], impressions: [], position: [], ctr: [] }; + stats.forEach((item) => { + chartSeries.clicks.push(item.clicks); + chartSeries.impressions.push(item.impressions); + chartSeries.position.push(item.position); + chartSeries.ctr.push(item.ctr); + }); + return { + labels: stats && stats.length > 0 ? stats.map((item) => `${new Date(item.date).getDate()}-${months[new Date(item.date).getMonth()]}`) : [], + series: chartSeries }; + }, [stats]); + + const renderChart = () => { + // Doc: https://www.chartjs.org/docs/latest/samples/line/multi-axis.html + const chartOptions = { + responsive: true, + maintainAspectRatio: false, + animation: false as const, + interaction: { + mode: 'index' as const, + intersect: false, + }, + scales: { + x: { + grid: { + drawOnChartArea: false, + }, + }, + y1: { + display: true, + position: 'right' as const, + grid: { + drawOnChartArea: false, + }, + }, + }, + plugins: { + legend: { + display: false, + }, + }, + }; + const { clicks, impressions } = chartData.series || {}; + const dataSet = [ + { label: 'Visits', data: clicks, borderColor: 'rgb(117, 50, 205)', backgroundColor: 'rgba(117, 50, 205, 0.5)', yAxisID: 'y' }, + { label: 'Impressions', data: impressions, borderColor: 'rgb(31, 205, 176)', backgroundColor: 'rgba(31, 205, 176, 0.5)', yAxisID: 'y1' }, + ]; + return ; + }; + + return ( +
    +
    +
    + Visits + {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(totalStat.clicks || 0).replace('T', 'K')} +
    +
    + Impressions + {new Intl.NumberFormat('en-US', { notation: 'compact', compactDisplay: 'short' }).format(totalStat.impressions || 0).replace('T', 'K')} +
    +
    + Avg Position + {(totalStat.position ? Math.round(totalStat.position / stats.length) : 0)} +
    +
    + Avg CTR + {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(totalStat.ctr || 0)}% +
    +
    + Keywords + {formattedNum(totalKeywords)} +
    +
    + Pages + {formattedNum(totalPages)} +
    +
    +
    + {renderChart()} +
    +
    + ); +}; + +export default InsightStats; diff --git a/components/keywords/AddKeywords.tsx b/components/keywords/AddKeywords.tsx index 2e0340e..b26b0ce 100644 --- a/components/keywords/AddKeywords.tsx +++ b/components/keywords/AddKeywords.tsx @@ -34,7 +34,9 @@ const AddKeywords = ({ closeModal, domain, keywords }: AddKeywordsProps) => { setError(`Keywords ${keywordExist.join(',')} already Exist`); setTimeout(() => { setError(''); }, 3000); } else { - addMutate({ ...newKeywordsData, keywords: keywordsArray.join('\n') }); + const { device, country, domain: kDomain, tags } = newKeywordsData; + const newKeywordsArray = keywordsArray.map((nItem) => ({ keyword: nItem, device, country, domain: kDomain, tags })); + addMutate(newKeywordsArray); } } else { setError('Please Insert a Keyword'); diff --git a/components/keywords/Keyword.tsx b/components/keywords/Keyword.tsx index 9222c41..7c14f35 100644 --- a/components/keywords/Keyword.tsx +++ b/components/keywords/Keyword.tsx @@ -15,11 +15,25 @@ type KeywordProps = { selectKeyword: Function, manageTags: Function, showKeywordDetails: Function, - lastItem?:boolean + lastItem?:boolean, + showSCData: boolean, + scDataType: string, } const Keyword = (props: KeywordProps) => { - const { keywordData, refreshkeyword, favoriteKeyword, removeKeyword, selectKeyword, selected, showKeywordDetails, manageTags, lastItem } = props; + const { + keywordData, + refreshkeyword, + favoriteKeyword, + removeKeyword, + selectKeyword, + selected, + showKeywordDetails, + manageTags, + lastItem, + showSCData = true, + scDataType = 'threeDays', + } = props; const { keyword, domain, ID, position, url = '', lastUpdated, country, sticky, history = {}, updating = false, lastUpdateError = false, } = keywordData; @@ -48,14 +62,14 @@ const Keyword = (props: KeywordProps) => { const optionsButtonStyle = 'block px-2 py-2 cursor-pointer hover:bg-indigo-50 hover:text-blue-700'; - const renderPosition = () => { - if (position === 0) { + const renderPosition = (pos:number, type?:string) => { + if (pos === 0) { return {'>100'}; } - if (updating) { + if (updating && type !== 'sc') { return ; } - return position; + return pos; }; return ( @@ -86,7 +100,7 @@ const Keyword = (props: KeywordProps) => {
    - {renderPosition()} + {renderPosition(position)} {!updating && positionChange > 0 && ▲ {positionChange}} {!updating && positionChange < 0 && ▼ {positionChange}}
    @@ -104,7 +118,25 @@ const Keyword = (props: KeywordProps) => { -
    + + {showSCData && ( +
    + + SC Position: + {renderPosition(keywordData?.scData?.position[scDataType as keyof KeywordSCDataChild] || 0, 'sc')} + + + Impressions: {keywordData?.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0} + + + Visits: {keywordData?.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0} + + {/* {keywordData?.scData?.ctr[scDataType] || '0.00%'} */} +
    + )} + +
    + + {keyword} + +
    + +
    + {renderPosition()} + Position +
    + +
    + + + + {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(impressions)} +
    + +
    + + + + {new Intl.NumberFormat('en-IN', { maximumSignificantDigits: 3 }).format(clicks)} +
    + +
    + + + + {new Intl.NumberFormat('en-US', { minimumFractionDigits: 2, maximumFractionDigits: 2 }).format(ctr)}% +
    + +
    + ); + }; + + export default SCKeyword; diff --git a/components/keywords/SCKeywordsTable.tsx b/components/keywords/SCKeywordsTable.tsx new file mode 100644 index 0000000..40a1e6f --- /dev/null +++ b/components/keywords/SCKeywordsTable.tsx @@ -0,0 +1,227 @@ +import { useRouter } from 'next/router'; +import React, { useState, useMemo, useEffect } from 'react'; +import { Toaster } from 'react-hot-toast'; +import { FixedSizeList as List, ListChildComponentProps } from 'react-window'; +import { useAddKeywords } from '../../services/keywords'; +import { SCfilterKeywords, SCkeywordsByDevice, SCsortKeywords } from '../../utils/SCsortFilter'; +import Icon from '../common/Icon'; +import KeywordFilters from './KeywordFilter'; +import SCKeyword from './SCKeyword'; + +type SCKeywordsTableProps = { + domain: DomainType | null, + keywords: SearchAnalyticsItem[], + isLoading: boolean, + isConsoleIntegrated: boolean, +} + +type SCCountryDataType = { + keywords: number, + impressions: number, + visits: number +} + +const SCKeywordsTable = ({ domain, keywords = [], isLoading = true, isConsoleIntegrated = true }: SCKeywordsTableProps) => { + const router = useRouter(); + const [device, setDevice] = useState('desktop'); + const [selectedKeywords, setSelectedKeywords] = useState([]); + const [filterParams, setFilterParams] = useState({ countries: [], tags: [], search: '' }); + const [sortBy, setSortBy] = useState('imp_asc'); + const [isMobile, setIsMobile] = useState(false); + const [SCListHeight, setSCListHeight] = useState(500); + + const { mutate: addKeywords } = useAddKeywords(() => { if (domain && domain.slug) router.push(`/domain/${domain.slug}`); }); + const finalKeywords: {[key:string] : SCKeywordType[] } = useMemo(() => { + const procKeywords = keywords.filter((x) => x.device === device); + const filteredKeywords = SCfilterKeywords(procKeywords, filterParams); + const sortedKeywords = SCsortKeywords(filteredKeywords, sortBy); + return SCkeywordsByDevice(sortedKeywords, device); + }, [keywords, device, filterParams, sortBy]); + + const SCCountryData: {[key:string] : SCCountryDataType } = useMemo(() => { + const countryData:{[key:string] : SCCountryDataType } = {}; + + Object.keys(finalKeywords).forEach((dateKey) => { + finalKeywords[dateKey].forEach((keyword) => { + const kCountry = keyword.country; + if (!countryData[kCountry]) { countryData[kCountry] = { keywords: 0, impressions: 0, visits: 0 }; } + countryData[kCountry].keywords += 1; + countryData[kCountry].visits += (keyword.clicks || 0); + countryData[kCountry].impressions += (keyword.impressions || 0); + }); + }); + + return countryData; + }, [finalKeywords]); + + const viewSummary: {[key:string] : number } = useMemo(() => { + const keyCount = finalKeywords[device].length; + const kwSummary = { position: 0, impressions: 0, visits: 0, ctr: 0 }; + finalKeywords[device].forEach((k) => { + kwSummary.position += k.position; + kwSummary.impressions += k.impressions; + kwSummary.visits += k.clicks; + kwSummary.ctr += k.ctr; + }); + return { + ...kwSummary, + position: Math.round(kwSummary.position / keyCount), + ctr: kwSummary.ctr / keyCount, + }; + }, [finalKeywords, device]); + + useEffect(() => { + setIsMobile(!!(window.matchMedia('only screen and (max-width: 760px)').matches)); + const resizeList = () => setSCListHeight(window.innerHeight - (isMobile ? 200 : 400)); + resizeList(); + window.addEventListener('resize', resizeList); + return () => { + window.removeEventListener('resize', resizeList); + }; + }, [isMobile]); + + const selectKeyword = (keywordID: string) => { + console.log('Select Keyword: ', keywordID); + let updatedSelectd = [...selectedKeywords, keywordID]; + if (selectedKeywords.includes(keywordID)) { + updatedSelectd = selectedKeywords.filter((keyID) => keyID !== keywordID); + } + setSelectedKeywords(updatedSelectd); + }; + + const addSCKeywordsToTracker = () => { + const selectedkeywords:KeywordAddPayload[] = []; + keywords.forEach((kitem:SCKeywordType) => { + if (selectedKeywords.includes(kitem.uid)) { + const { keyword, country } = kitem; + selectedkeywords.push({ keyword, device, country, domain: domain?.domain || '', tags: '' }); + } + }); + addKeywords(selectedkeywords); + setSelectedKeywords([]); + }; + + const selectedAllItems = selectedKeywords.length === finalKeywords[device].length; + + const Row = ({ data, index, style }:ListChildComponentProps) => { + const keyword = data[index]; + return ( + + ); +}; + + return ( +
    +
    + {selectedKeywords.length > 0 && ( + + )} + {selectedKeywords.length === 0 && ( + setFilterParams(params)} + updateSort={(sorted:string) => setSortBy(sorted)} + sortBy={sortBy} + keywords={keywords} + device={device} + setDevice={setDevice} + isConsole={true} + integratedConsole={isConsoleIntegrated} + SCcountries={Object.keys(SCCountryData)} + /> + )} +
    +
    + +
    + {!isLoading && finalKeywords[device] && finalKeywords[device].length > 0 && ( + + {Row} + + )} + {!isLoading && finalKeywords[device] && finalKeywords[device].length > 0 && ( + + )} + {isConsoleIntegrated && !isLoading && finalKeywords[device].length === 0 && ( +

    + Could Not fetch Keyword Data for this Domain from Google Search Console. +

    + )} + {isConsoleIntegrated && isLoading && ( +

    Loading Keywords...

    + )} + {!isConsoleIntegrated && ( +

    + Goolge Search has not been Integrated yet. Please See the Docs to Learn how to integrate Google Search Data for this Domain. +

    + )} +
    +
    +
    +
    + +
    + ); + }; + + export default SCKeywordsTable; diff --git a/cron.js b/cron.js index 5a18e39..6c2c1f8 100644 --- a/cron.js +++ b/cron.js @@ -101,6 +101,21 @@ const runAppCronJobs = () => { }); }, { scheduled: true }); + // Run Google Search Console Scraper Daily + if (process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) { + const searchConsoleCRONTime = generateCronTime('daily'); + cron.schedule(searchConsoleCRONTime, () => { + const fetchOpts = { method: 'POST', headers: { Authorization: `Bearer ${process.env.APIKEY}` } }; + fetch(`${process.env.NEXT_PUBLIC_APP_URL}/api/searchconsole`, fetchOpts) + .then((res) => res.json()) + .then((data) => console.log(data)) + .catch((err) => { + console.log('ERROR Making Google Search Console Scraper Cron Request..'); + console.log(err); + }); + }, { scheduled: true }); + } + // RUN Email Notification CRON getAppSettings().then((settings) => { const notif_interval = (!settings.notification_interval || settings.notification_interval === 'never') ? false : settings.notification_interval; diff --git a/email/email.html b/email/email.html index 37c5c49..812b4cd 100644 --- a/email/email.html +++ b/email/email.html @@ -259,13 +259,26 @@ color: #fff; border-radius: 4px 4px 0 0; } - .mainhead td { + .mainhead td, .subhead td { padding: 15px; } .mainhead a{ color: white; text-decoration: none; } + .subhead { + background: #dee3ff; + color: #344dd7; + border-radius: 4px 4px 0 0; + } + .subhead a { + color: #344dd7; + text-decoration: none; + } + .console_table{ + margin-top: 40px; + margin-bottom: 20px; + } .keyword_table td { padding: 10px 0; } @@ -277,13 +290,16 @@ .keyword td:nth-child(1){ font-weight: bold; } - .keyword_table th:nth-child(2), .keyword_table th:nth-child(3), .keyword td:nth-child(2), .keyword td:nth-child(3){ + .keyword_table th:nth-child(2), .keyword_table th:nth-child(3), .keyword td:nth-child(2), .keyword td:nth-child(3), .keyword_table--sc th:nth-child(4), .keyword_table--sc td:nth-child(4){ text-align: center; } .keyword td:nth-child(3){ font-size: 12px; color: #888; } + .keyword_table--sc td:nth-child(3){ + color:inherit; + } .keyword svg { width: 15px; } @@ -304,6 +320,9 @@ vertical-align: middle; opacity: 0.6; } + .google_icon{ + max-width: 13px; + } /* ------------------------------------- RESPONSIVE AND MOBILE FRIENDLY STYLES ------------------------------------- */ @@ -437,6 +456,9 @@ + + {{SCStatsTable}} + diff --git a/package-lock.json b/package-lock.json index 3e6330e..88edba4 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,6 +8,7 @@ "name": "serpbear", "version": "0.1.7", "dependencies": { + "@googleapis/searchconsole": "^1.0.0", "@testing-library/react": "^13.4.0", "@types/react-transition-group": "^4.4.5", "axios": "^1.1.3", @@ -33,6 +34,7 @@ "react-query": "^3.39.2", "react-timeago": "^7.1.0", "react-transition-group": "^4.4.5", + "react-window": "^1.8.8", "reflect-metadata": "^0.1.13", "sequelize": "^6.25.2", "sequelize-typescript": "^2.1.5", @@ -49,6 +51,7 @@ "@types/react": "18.0.21", "@types/react-dom": "18.0.6", "@types/react-timeago": "^4.1.3", + "@types/react-window": "^1.8.5", "autoprefixer": "^10.4.12", "eslint": "8.25.0", "eslint-config-airbnb-base": "^15.0.0", @@ -755,6 +758,17 @@ "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", "optional": true }, + "node_modules/@googleapis/searchconsole": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@googleapis/searchconsole/-/searchconsole-1.0.0.tgz", + "integrity": "sha512-ssdO6oQyS+AuZHJZY50aCso7orPPRR9Y9lgdAtzRh9VJdk8L/3nHz8ySwb/6AaF+FPK3PfkvGFPRsenvLe3S+A==", + "dependencies": { + "googleapis-common": "^6.0.3" + }, + "engines": { + "node": ">=12.0.0" + } + }, "node_modules/@humanwhocodes/config-array": { "version": "0.10.7", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.7.tgz", @@ -2056,6 +2070,15 @@ "@types/react": "*" } }, + "node_modules/@types/react-window": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@types/react-window/-/react-window-1.8.5.tgz", + "integrity": "sha512-V9q3CvhC9Jk9bWBOysPGaWy/Z0lxYcTXLtLipkt2cnRj1JOSFNF7wqGpkScSXMgBwC+fnVRg/7shwgddBG5ICw==", + "dev": true, + "dependencies": { + "@types/react": "*" + } + }, "node_modules/@types/scheduler": { "version": "0.16.2", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", @@ -2780,6 +2803,14 @@ "node": ">=0.6" } }, + "node_modules/bignumber.js": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.1.tgz", + "integrity": "sha512-pHm4LsMJ6lzgNGVfZHjMoO8sdoRhOzOH4MLmY65Jg70bpxCKu5iOHNJyfF6OyvYw7t8Fpf35RuzUyqnQsj8Vig==", + "engines": { + "node": "*" + } + }, "node_modules/binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -5094,6 +5125,11 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "node_modules/external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -5164,6 +5200,11 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "node_modules/fast-text-encoding": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", + "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==" + }, "node_modules/fastest-levenshtein": { "version": "1.0.16", "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", @@ -5404,6 +5445,32 @@ "node": ">=10" } }, + "node_modules/gaxios": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz", + "integrity": "sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og==", + "dependencies": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.7" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/gcp-metadata": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.1.0.tgz", + "integrity": "sha512-QVjouEXvNVG/nde6VZDXXFTB02xQdztaumkWCHUff58qsdCS05/8OPh68fQ2QnArfAzZTwfEc979FHSHsU8EWg==", + "dependencies": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + }, + "engines": { + "node": ">=12" + } + }, "node_modules/gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -5761,6 +5828,90 @@ "csstype": "^3.0.10" } }, + "node_modules/google-auth-library": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz", + "integrity": "sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q==", + "dependencies": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^5.0.0", + "gcp-metadata": "^5.0.0", + "gtoken": "^6.1.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + }, + "engines": { + "node": ">=12" + } + }, + "node_modules/google-auth-library/node_modules/arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==", + "engines": { + "node": ">=8" + } + }, + "node_modules/google-auth-library/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/google-auth-library/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/google-p12-pem": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", + "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==", + "dependencies": { + "node-forge": "^1.3.1" + }, + "bin": { + "gp12-pem": "build/src/bin/gp12-pem.js" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/googleapis-common": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-6.0.4.tgz", + "integrity": "sha512-m4ErxGE8unR1z0VajT6AYk3s6a9gIMM6EkDZfkPnES8joeOlEtFEJeF8IyZkb0tjPXkktUfYrE4b3Li1DNyOwA==", + "dependencies": { + "extend": "^3.0.2", + "gaxios": "^5.0.1", + "google-auth-library": "^8.0.2", + "qs": "^6.7.0", + "url-template": "^2.0.8", + "uuid": "^9.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/googleapis-common/node_modules/uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -5792,6 +5943,38 @@ "node": "^12.22.0 || ^14.16.0 || ^16.0.0 || >=17.0.0" } }, + "node_modules/gtoken": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", + "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==", + "dependencies": { + "gaxios": "^5.0.1", + "google-p12-pem": "^4.0.0", + "jws": "^4.0.0" + }, + "engines": { + "node": ">=12.0.0" + } + }, + "node_modules/gtoken/node_modules/jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "dependencies": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/gtoken/node_modules/jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "dependencies": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + }, "node_modules/handlebars": { "version": "4.7.7", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", @@ -6484,7 +6667,6 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true, "engines": { "node": ">=8" }, @@ -7697,6 +7879,14 @@ "node": ">=4" } }, + "node_modules/json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "dependencies": { + "bignumber.js": "^9.0.0" + } + }, "node_modules/json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -8182,6 +8372,11 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/memoize-one": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", + "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==" + }, "node_modules/meow": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/meow/-/meow-8.1.2.tgz", @@ -8818,6 +9013,14 @@ } } }, + "node_modules/node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==", + "engines": { + "node": ">= 6.13.0" + } + }, "node_modules/node-gyp": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", @@ -9745,6 +9948,20 @@ "teleport": ">=0.2.0" } }, + "node_modules/qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", @@ -9883,6 +10100,22 @@ "react-dom": ">=16.6.0" } }, + "node_modules/react-window": { + "version": "1.8.8", + "resolved": "https://registry.npmjs.org/react-window/-/react-window-1.8.8.tgz", + "integrity": "sha512-D4IiBeRtGXziZ1n0XklnFGu7h9gU684zepqyKzgPNzrsrk7xOCxni+TCckjg2Nr/DiaEEGVVmnhYSlT2rB47dQ==", + "dependencies": { + "@babel/runtime": "^7.0.0", + "memoize-one": ">=3.1.1 <6" + }, + "engines": { + "node": ">8.0.0" + }, + "peerDependencies": { + "react": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0", + "react-dom": "^15.0.0 || ^16.0.0 || ^17.0.0 || ^18.0.0" + } + }, "node_modules/read-cache": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", @@ -11835,6 +12068,11 @@ "requires-port": "^1.0.0" } }, + "node_modules/url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==" + }, "node_modules/use-sync-external-store": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", @@ -12742,6 +12980,14 @@ "integrity": "sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==", "optional": true }, + "@googleapis/searchconsole": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@googleapis/searchconsole/-/searchconsole-1.0.0.tgz", + "integrity": "sha512-ssdO6oQyS+AuZHJZY50aCso7orPPRR9Y9lgdAtzRh9VJdk8L/3nHz8ySwb/6AaF+FPK3PfkvGFPRsenvLe3S+A==", + "requires": { + "googleapis-common": "^6.0.3" + } + }, "@humanwhocodes/config-array": { "version": "0.10.7", "resolved": "https://registry.npmjs.org/@humanwhocodes/config-array/-/config-array-0.10.7.tgz", @@ -13769,6 +14015,15 @@ "@types/react": "*" } }, + "@types/react-window": { + "version": "1.8.5", + "resolved": "https://registry.npmjs.org/@types/react-window/-/react-window-1.8.5.tgz", + "integrity": "sha512-V9q3CvhC9Jk9bWBOysPGaWy/Z0lxYcTXLtLipkt2cnRj1JOSFNF7wqGpkScSXMgBwC+fnVRg/7shwgddBG5ICw==", + "dev": true, + "requires": { + "@types/react": "*" + } + }, "@types/scheduler": { "version": "0.16.2", "resolved": "https://registry.npmjs.org/@types/scheduler/-/scheduler-0.16.2.tgz", @@ -14292,6 +14547,11 @@ "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.51.tgz", "integrity": "sha512-GPEid2Y9QU1Exl1rpO9B2IPJGHPSupF5GnVIP0blYvNOMer2bTvSWs1jGOUg04hTmu67nmLsQ9TBo1puaotBHg==" }, + "bignumber.js": { + "version": "9.1.1", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.1.1.tgz", + "integrity": "sha512-pHm4LsMJ6lzgNGVfZHjMoO8sdoRhOzOH4MLmY65Jg70bpxCKu5iOHNJyfF6OyvYw7t8Fpf35RuzUyqnQsj8Vig==" + }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -16030,6 +16290,11 @@ "jest-util": "^29.3.1" } }, + "extend": { + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", + "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" + }, "external-editor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/external-editor/-/external-editor-3.1.0.tgz", @@ -16092,6 +16357,11 @@ "integrity": "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw==", "dev": true }, + "fast-text-encoding": { + "version": "1.0.6", + "resolved": "https://registry.npmjs.org/fast-text-encoding/-/fast-text-encoding-1.0.6.tgz", + "integrity": "sha512-VhXlQgj9ioXCqGstD37E/HBeqEGV/qOD/kmbVG8h5xKBYvM1L3lR1Zn4555cQ8GkYbJa8aJSipLPndE1k6zK2w==" + }, "fastest-levenshtein": { "version": "1.0.16", "resolved": "https://registry.npmjs.org/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz", @@ -16261,6 +16531,26 @@ "wide-align": "^1.1.2" } }, + "gaxios": { + "version": "5.0.2", + "resolved": "https://registry.npmjs.org/gaxios/-/gaxios-5.0.2.tgz", + "integrity": "sha512-TjtV2AJOZoMQqRYoy5eM8cCQogYwazWNYLQ72QB0kwa6vHHruYkGmhhyrlzbmgNHK1dNnuP2WSH81urfzyN2Og==", + "requires": { + "extend": "^3.0.2", + "https-proxy-agent": "^5.0.0", + "is-stream": "^2.0.0", + "node-fetch": "^2.6.7" + } + }, + "gcp-metadata": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/gcp-metadata/-/gcp-metadata-5.1.0.tgz", + "integrity": "sha512-QVjouEXvNVG/nde6VZDXXFTB02xQdztaumkWCHUff58qsdCS05/8OPh68fQ2QnArfAzZTwfEc979FHSHsU8EWg==", + "requires": { + "gaxios": "^5.0.0", + "json-bigint": "^1.0.0" + } + }, "gensync": { "version": "1.0.0-beta.2", "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", @@ -16538,6 +16828,76 @@ "integrity": "sha512-5SS2lmxbhqH0u9ABEWq7WPU69a4i2pYcHeCxqaNq6Cw3mnrF0ghWNM4tEGid4dKy8XNIAUbuThuozDHHKJVh3A==", "requires": {} }, + "google-auth-library": { + "version": "8.7.0", + "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-8.7.0.tgz", + "integrity": "sha512-1M0NG5VDIvJZEnstHbRdckLZESoJwguinwN8Dhae0j2ZKIQFIV63zxm6Fo6nM4xkgqUr2bbMtV5Dgo+Hy6oo0Q==", + "requires": { + "arrify": "^2.0.0", + "base64-js": "^1.3.0", + "ecdsa-sig-formatter": "^1.0.11", + "fast-text-encoding": "^1.0.0", + "gaxios": "^5.0.0", + "gcp-metadata": "^5.0.0", + "gtoken": "^6.1.0", + "jws": "^4.0.0", + "lru-cache": "^6.0.0" + }, + "dependencies": { + "arrify": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/arrify/-/arrify-2.0.1.tgz", + "integrity": "sha512-3duEwti880xqi4eAMN8AyR4a0ByT90zoYdLlevfrvU43vb0YZwZVfxOgxWrLXXXpyugL0hNZc9G6BiB5B3nUug==" + }, + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + } + } + }, + "google-p12-pem": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/google-p12-pem/-/google-p12-pem-4.0.1.tgz", + "integrity": "sha512-WPkN4yGtz05WZ5EhtlxNDWPhC4JIic6G8ePitwUWy4l+XPVYec+a0j0Ts47PDtW59y3RwAhUd9/h9ZZ63px6RQ==", + "requires": { + "node-forge": "^1.3.1" + } + }, + "googleapis-common": { + "version": "6.0.4", + "resolved": "https://registry.npmjs.org/googleapis-common/-/googleapis-common-6.0.4.tgz", + "integrity": "sha512-m4ErxGE8unR1z0VajT6AYk3s6a9gIMM6EkDZfkPnES8joeOlEtFEJeF8IyZkb0tjPXkktUfYrE4b3Li1DNyOwA==", + "requires": { + "extend": "^3.0.2", + "gaxios": "^5.0.1", + "google-auth-library": "^8.0.2", + "qs": "^6.7.0", + "url-template": "^2.0.8", + "uuid": "^9.0.0" + }, + "dependencies": { + "uuid": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-9.0.0.tgz", + "integrity": "sha512-MXcSTerfPa4uqyzStbRoTgt5XIe3x5+42+q1sDuy3R5MDk66URdLMOZe5aPX/SQd+kuYAh0FdP/pO28IkQyTeg==" + } + } + }, "gopd": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", @@ -16563,6 +16923,37 @@ "resolved": "https://registry.npmjs.org/graphql/-/graphql-16.6.0.tgz", "integrity": "sha512-KPIBPDlW7NxrbT/eh4qPXz5FiFdL5UbaA0XUNz2Rp3Z3hqBSkbj0GVjwFDztsWVauZUWsbKHgMg++sk8UX0bkw==" }, + "gtoken": { + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/gtoken/-/gtoken-6.1.2.tgz", + "integrity": "sha512-4ccGpzz7YAr7lxrT2neugmXQ3hP9ho2gcaityLVkiUecAiwiy60Ii8gRbZeOsXV19fYaRjgBSshs8kXw+NKCPQ==", + "requires": { + "gaxios": "^5.0.1", + "google-p12-pem": "^4.0.0", + "jws": "^4.0.0" + }, + "dependencies": { + "jwa": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.0.tgz", + "integrity": "sha512-jrZ2Qx916EA+fq9cEAeCROWPTfCwi1IVHqT2tapuqLEVVDKFDENFw1oL+MwrTvH6msKxsd1YTDVw6uKEcsrLEA==", + "requires": { + "buffer-equal-constant-time": "1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "jws": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.0.tgz", + "integrity": "sha512-KDncfTmOZoOMTFG4mBlG0qUIOlc03fmzH+ru6RgYVZhPkyiy/92Owlt/8UEN+a4TXR1FQetfIpJE8ApdvdVxTg==", + "requires": { + "jwa": "^2.0.0", + "safe-buffer": "^5.0.1" + } + } + } + }, "handlebars": { "version": "4.7.7", "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.7.tgz", @@ -17039,8 +17430,7 @@ "is-stream": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/is-stream/-/is-stream-2.0.1.tgz", - "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==", - "dev": true + "integrity": "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==" }, "is-string": { "version": "1.0.7", @@ -17959,6 +18349,14 @@ "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", "dev": true }, + "json-bigint": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-1.0.0.tgz", + "integrity": "sha512-SiPv/8VpZuWbvLSMtTDU8hEfrZWg/mH/nV/b4o0CYbSxu1UIQPLdwKOCIyLQX+VIPO5vrLX3i8qtqFyhdPSUSQ==", + "requires": { + "bignumber.js": "^9.0.0" + } + }, "json-parse-better-errors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/json-parse-better-errors/-/json-parse-better-errors-1.0.2.tgz", @@ -18354,6 +18752,11 @@ "dev": true, "peer": true }, + "memoize-one": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/memoize-one/-/memoize-one-5.2.1.tgz", + "integrity": "sha512-zYiwtZUcYyXKo/np96AGZAckk+FWWsUdJ3cHGGmld7+AhvcWmQyGCYUh1hc4Q/pkOhb65dQR/pqCyK0cOaHz4Q==" + }, "meow": { "version": "8.1.2", "resolved": "https://registry.npmjs.org/meow/-/meow-8.1.2.tgz", @@ -18794,6 +19197,11 @@ "whatwg-url": "^5.0.0" } }, + "node-forge": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/node-forge/-/node-forge-1.3.1.tgz", + "integrity": "sha512-dPEtOeMvF9VMcYV/1Wb8CPoVAXtp6MKMlcbAt4ddqmGqUJ6fQZFXkNZNkNlfevtNkGtaSoXf/vNNNSvgrdXwtA==" + }, "node-gyp": { "version": "8.4.1", "resolved": "https://registry.npmjs.org/node-gyp/-/node-gyp-8.4.1.tgz", @@ -19446,6 +19854,14 @@ "integrity": "sha512-kV/CThkXo6xyFEZUugw/+pIOywXcDbFYgSct5cT3gqlbkBE1SJdwy6UQoZvodiWF/ckQLZyDE/Bu1M6gVu5lVw==", "dev": true }, + "qs": { + "version": "6.11.0", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.11.0.tgz", + "integrity": "sha512-MvjoMCJwEarSbUYk5O+nmoSzSutSsTwF85zcHPQ9OrlFoZOYIjaqBAJIqIXjptyD5vThxGq52Xu/MaJzRkIk4Q==", + "requires": { + "side-channel": "^1.0.4" + } + }, "querystringify": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/querystringify/-/querystringify-2.2.0.tgz", @@ -19527,6 +19943,15 @@ "prop-types": "^15.6.2" } }, + "react-window": { + "version": "1.8.8", + "resolved": "https://registry.npmjs.org/react-window/-/react-window-1.8.8.tgz", + "integrity": "sha512-D4IiBeRtGXziZ1n0XklnFGu7h9gU684zepqyKzgPNzrsrk7xOCxni+TCckjg2Nr/DiaEEGVVmnhYSlT2rB47dQ==", + "requires": { + "@babel/runtime": "^7.0.0", + "memoize-one": ">=3.1.1 <6" + } + }, "read-cache": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/read-cache/-/read-cache-1.0.0.tgz", @@ -20991,6 +21416,11 @@ "requires-port": "^1.0.0" } }, + "url-template": { + "version": "2.0.8", + "resolved": "https://registry.npmjs.org/url-template/-/url-template-2.0.8.tgz", + "integrity": "sha512-XdVKMF4SJ0nP/O7XIPB0JwAEuT9lDIYnNsK8yGVe43y0AWoKeJNdv3ZNWh7ksJ6KqQFjOO6ox/VEitLnaVNufw==" + }, "use-sync-external-store": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz", diff --git a/package.json b/package.json index c25ad98..218ef8f 100644 --- a/package.json +++ b/package.json @@ -9,12 +9,14 @@ "cron": "node cron.js", "start:all": "concurrently npm:start npm:cron", "lint": "next lint", + "lint:css": "stylelint styles/*.css", "test": "jest --watch --verbose", "test:ci": "jest --ci", "test:cv": "jest --coverage --coverageDirectory='coverage'", "release": "standard-version" }, "dependencies": { + "@googleapis/searchconsole": "^1.0.0", "@testing-library/react": "^13.4.0", "@types/react-transition-group": "^4.4.5", "axios": "^1.1.3", @@ -40,6 +42,7 @@ "react-query": "^3.39.2", "react-timeago": "^7.1.0", "react-transition-group": "^4.4.5", + "react-window": "^1.8.8", "reflect-metadata": "^0.1.13", "sequelize": "^6.25.2", "sequelize-typescript": "^2.1.5", @@ -56,6 +59,7 @@ "@types/react": "18.0.21", "@types/react-dom": "18.0.6", "@types/react-timeago": "^4.1.3", + "@types/react-window": "^1.8.5", "autoprefixer": "^10.4.12", "eslint": "8.25.0", "eslint-config-airbnb-base": "^15.0.0", diff --git a/pages/_app.tsx b/pages/_app.tsx index b2c98e3..d8cd91a 100644 --- a/pages/_app.tsx +++ b/pages/_app.tsx @@ -5,7 +5,13 @@ import { QueryClient, QueryClientProvider } from 'react-query'; import { ReactQueryDevtools } from 'react-query/devtools'; function MyApp({ Component, pageProps }: AppProps) { - const [queryClient] = React.useState(() => new QueryClient()); + const [queryClient] = React.useState(() => new QueryClient({ + defaultOptions: { + queries: { + refetchOnWindowFocus: false, + }, + }, + })); return diff --git a/pages/api/domains.ts b/pages/api/domains.ts index e37483f..796ffe8 100644 --- a/pages/api/domains.ts +++ b/pages/api/domains.ts @@ -2,10 +2,11 @@ import type { NextApiRequest, NextApiResponse } from 'next'; import db from '../../database/database'; import Domain from '../../database/models/domain'; import Keyword from '../../database/models/keyword'; +import getdomainStats from '../../utils/domains'; import verifyUser from '../../utils/verifyUser'; type DomainsGetRes = { - domains: Domain[] + domains: DomainType[] error?: string|null, } @@ -47,9 +48,12 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) } export const getDomains = async (req: NextApiRequest, res: NextApiResponse) => { + const withStats = !!req?.query?.withstats; try { const allDomains: Domain[] = await Domain.findAll(); - return res.status(200).json({ domains: allDomains }); + const formattedDomains: DomainType[] = allDomains.map((el) => el.get({ plain: true })); + const theDomains: DomainType[] = withStats ? await getdomainStats(formattedDomains) : allDomains; + return res.status(200).json({ domains: theDomains }); } catch (error) { return res.status(400).json({ domains: [], error: 'Error Getting Domains.' }); } diff --git a/pages/api/insight.ts b/pages/api/insight.ts new file mode 100644 index 0000000..cadda4e --- /dev/null +++ b/pages/api/insight.ts @@ -0,0 +1,54 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import db from '../../database/database'; +import { getCountryInsight, getKeywordsInsight, getPagesInsight } from '../../utils/insight'; +import { fetchDomainSCData, readLocalSCData } from '../../utils/searchConsole'; +import verifyUser from '../../utils/verifyUser'; + +type SCInsightRes = { + data: InsightDataType | null, + error?: string|null, +} + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + await db.sync(); + const authorized = verifyUser(req, res); + if (authorized !== 'authorized') { + return res.status(401).json({ error: authorized }); + } + if (req.method === 'GET') { + return getDomainSearchConsoleInsight(req, res); + } + return res.status(502).json({ error: 'Unrecognized Route.' }); +} + +const getDomainSearchConsoleInsight = async (req: NextApiRequest, res: NextApiResponse) => { + if (!req.query.domain && typeof req.query.domain !== 'string') return res.status(400).json({ data: null, error: 'Domain is Missing.' }); + if (!!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) === false) { + return res.status(200).json({ data: null, error: 'Google Search Console Not Integrated' }); + } + const domainname = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-'); + const getInsightFromSCData = (localSCData: SCDomainDataType): InsightDataType => { + const { stats = [] } = localSCData; + const countries = getCountryInsight(localSCData); + const keywords = getKeywordsInsight(localSCData); + const pages = getPagesInsight(localSCData); + return { pages, keywords, countries, stats }; + }; + + // First try and read the Local SC Domain Data file. + const localSCData = await readLocalSCData(domainname); + if (localSCData && localSCData.stats && localSCData.stats.length) { + const response = getInsightFromSCData(localSCData); + return res.status(200).json({ data: response }); + } + + // If the Local SC Domain Data file does not exist, fetch from Googel Search Console. + try { + const scData = await fetchDomainSCData(domainname); + const response = getInsightFromSCData(scData); + return res.status(200).json({ data: response }); + } catch (error) { + console.log('ERROR getDomainSearchConsoleInsight: ', error); + return res.status(400).json({ data: null, error: 'Error Fetching Stats from Google Search Console.' }); + } +}; diff --git a/pages/api/keywords.ts b/pages/api/keywords.ts index 13ea134..019ecbd 100644 --- a/pages/api/keywords.ts +++ b/pages/api/keywords.ts @@ -6,6 +6,7 @@ import { refreshAndUpdateKeywords } from './refresh'; import { getAppSettings } from './settings'; import verifyUser from '../../utils/verifyUser'; import parseKeywords from '../../utils/parseKeywords'; +import { integrateKeywordSCData, readLocalSCData } from '../../utils/searchConsole'; type KeywordsGetResponse = { keywords?: KeywordType[], @@ -45,11 +46,13 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse e.get({ plain: true }))); - const slimKeywords = keywords.map((keyword) => { + const processedKeywords = keywords.map((keyword) => { const historyArray = Object.keys(keyword.history).map((dateKey:string) => ({ date: new Date(dateKey).getTime(), dateRaw: dateKey, @@ -58,10 +61,12 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse a.date - b.date); const lastWeekHistory :KeywordHistory = {}; historySorted.slice(-7).forEach((x:any) => { lastWeekHistory[x.dateRaw] = x.position; }); - return { ...keyword, lastResult: [], history: lastWeekHistory }; + const keywordWithSlimHistory = { ...keyword, lastResult: [], history: lastWeekHistory }; + const finalKeyword = domainSCData ? integrateKeywordSCData(keyword, domainSCData) : keywordWithSlimHistory; + return finalKeyword; }); console.log('getKeywords: ', keywords.length); - return res.status(200).json({ keywords: slimKeywords }); + return res.status(200).json({ keywords: processedKeywords }); } catch (error) { console.log(error); return res.status(400).json({ error: 'Error Loading Keywords for this Domain.' }); @@ -69,13 +74,14 @@ const getKeywords = async (req: NextApiRequest, res: NextApiResponse) => { - const { keywords, device, country, domain, tags } = req.body; - if (keywords && device && country) { - const keywordsArray = keywords.replaceAll('\n', ',').split(',').map((item:string) => item.trim()); - const tagsArray = tags ? tags.split(',').map((item:string) => item.trim()) : []; + const { keywords } = req.body; + if (keywords && Array.isArray(keywords) && keywords.length > 0) { + // const keywordsArray = keywords.replaceAll('\n', ',').split(',').map((item:string) => item.trim()); const keywordsToAdd: any = []; // QuickFIX for bug: https://github.com/sequelize/sequelize-typescript/issues/936 - keywordsArray.forEach((keyword: string) => { + keywords.forEach((kwrd: KeywordAddPayload) => { + const { keyword, device, country, domain, tags } = kwrd; + const tagsArray = tags ? tags.split(',').map((item:string) => item.trim()) : []; const newKeyword = { keyword, device, diff --git a/pages/api/notify.ts b/pages/api/notify.ts index 70f1e1f..2368b0f 100644 --- a/pages/api/notify.ts +++ b/pages/api/notify.ts @@ -21,44 +21,28 @@ export default async function handler(req: NextApiRequest, res: NextApiResponse) } const notify = async (req: NextApiRequest, res: NextApiResponse) => { + const reqDomain = req?.query?.domain as string || ''; try { const settings = await getAppSettings(); - const { - smtp_server = '', - smtp_port = '', - smtp_username = '', - smtp_password = '', - notification_email = '', - notification_email_from = '', - } = settings; + const { smtp_server = '', smtp_port = '', smtp_username = '', smtp_password = '', notification_email = '' } = settings; if (!smtp_server || !smtp_port || !smtp_username || !smtp_password || !notification_email) { return res.status(401).json({ success: false, error: 'SMTP has not been setup properly!' }); } - const fromEmail = `SerpBear <${notification_email_from || 'no-reply@serpbear.com'}>`; - const transporter = nodeMailer.createTransport({ - host: smtp_server, - port: parseInt(smtp_port, 10), - auth: { user: smtp_username, pass: smtp_password }, - }); - const allDomains: Domain[] = await Domain.findAll(); - - if (allDomains && allDomains.length > 0) { - const domains = allDomains.map((el) => el.get({ plain: true })); - for (const domain of domains) { - if (domain.notification !== false) { - const query = { where: { domain: domain.domain } }; - const domainKeywords:Keyword[] = await Keyword.findAll(query); - const keywordsArray = domainKeywords.map((el) => el.get({ plain: true })); - const keywords: KeywordType[] = parseKeywords(keywordsArray); - await transporter.sendMail({ - from: fromEmail, - to: domain.notification_emails || notification_email, - subject: `[${domain.domain}] Keyword Positions Update`, - html: await generateEmail(domain.domain, keywords), - }); - // console.log(JSON.stringify(result, null, 4)); + if (reqDomain) { + const theDomain = await Domain.findOne({ where: { domain: reqDomain } }); + if (theDomain) { + await sendNotificationEmail(theDomain, settings); + } + } else { + const allDomains: Domain[] = await Domain.findAll(); + if (allDomains && allDomains.length > 0) { + const domains = allDomains.map((el) => el.get({ plain: true })); + for (const domain of domains) { + if (domain.notification !== false) { + await sendNotificationEmail(domain, settings); + } } } } @@ -69,3 +53,33 @@ const notify = async (req: NextApiRequest, res: NextApiResponse) return res.status(401).json({ success: false, error: 'Error Sending Notification Email.' }); } }; + +const sendNotificationEmail = async (domain: Domain, settings: SettingsType) => { + const { + smtp_server = '', + smtp_port = '', + smtp_username = '', + smtp_password = '', + notification_email = '', + notification_email_from = '', + } = settings; + + const fromEmail = `SerpBear <${notification_email_from || 'no-reply@serpbear.com'}>`; + const transporter = nodeMailer.createTransport({ + host: smtp_server, + port: parseInt(smtp_port, 10), + auth: { user: smtp_username, pass: smtp_password }, + }); + const domainName = domain.domain; + const query = { where: { domain: domainName } }; + const domainKeywords:Keyword[] = await Keyword.findAll(query); + const keywordsArray = domainKeywords.map((el) => el.get({ plain: true })); + const keywords: KeywordType[] = parseKeywords(keywordsArray); + const emailHTML = await generateEmail(domainName, keywords); + await transporter.sendMail({ + from: fromEmail, + to: domain.notification_emails || notification_email, + subject: `[${domainName}] Keyword Positions Update`, + html: emailHTML, + }); +}; diff --git a/pages/api/searchconsole.ts b/pages/api/searchconsole.ts new file mode 100644 index 0000000..04f453b --- /dev/null +++ b/pages/api/searchconsole.ts @@ -0,0 +1,65 @@ +import type { NextApiRequest, NextApiResponse } from 'next'; +import db from '../../database/database'; +import Domain from '../../database/models/domain'; +import { fetchDomainSCData, readLocalSCData } from '../../utils/searchConsole'; +import verifyUser from '../../utils/verifyUser'; + +type searchConsoleRes = { + data: SCDomainDataType|null + error?: string|null, +} + +type searchConsoleCRONRes = { + status: string, + error?: string|null, +} + +export default async function handler(req: NextApiRequest, res: NextApiResponse) { + await db.sync(); + const authorized = verifyUser(req, res); + if (authorized !== 'authorized') { + return res.status(401).json({ error: authorized }); + } + if (req.method === 'GET') { + return getDomainSearchConsoleData(req, res); + } + if (req.method === 'POST') { + return cronRefreshSearchConsoleData(req, res); + } + return res.status(502).json({ error: 'Unrecognized Route.' }); +} + +const getDomainSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse) => { + if (!req.query.domain && typeof req.query.domain !== 'string') return res.status(400).json({ data: null, error: 'Domain is Missing.' }); + if (!!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL) === false) { + return res.status(200).json({ data: null, error: 'Google Search Console Not Integrated' }); + } + const domainname = (req.query.domain as string).replaceAll('-', '.').replaceAll('_', '-'); + const localSCData = await readLocalSCData(domainname); + console.log(localSCData && localSCData.thirtyDays && localSCData.thirtyDays.length); + + if (localSCData && localSCData.thirtyDays && localSCData.thirtyDays.length) { + return res.status(200).json({ data: localSCData }); + } + try { + const scData = await fetchDomainSCData(domainname); + return res.status(200).json({ data: scData }); + } catch (error) { + console.log('ERROR getDomainSearchConsoleData: ', error); + return res.status(400).json({ data: null, error: 'Error Fetching Data from Google Search Console.' }); + } +}; + +const cronRefreshSearchConsoleData = async (req: NextApiRequest, res: NextApiResponse) => { + try { + const allDomainsRaw = await Domain.findAll(); + const Domains: Domain[] = allDomainsRaw.map((el) => el.get({ plain: true })); + for (const domain of Domains) { + await fetchDomainSCData(domain.domain); + } + return res.status(200).json({ status: 'completed' }); + } catch (error) { + console.log('ERROR cronRefreshkeywords: ', error); + return res.status(400).json({ status: 'failed', error: 'Error Fetching Data from Google Search Console.' }); + } +}; diff --git a/pages/api/settings.ts b/pages/api/settings.ts index de3c5b8..afa77b4 100644 --- a/pages/api/settings.ts +++ b/pages/api/settings.ts @@ -60,7 +60,12 @@ export const getAppSettings = async () : Promise => { const cryptr = new Cryptr(process.env.SECRET as string); const scaping_api = settings.scaping_api ? cryptr.decrypt(settings.scaping_api) : ''; const smtp_password = settings.smtp_password ? cryptr.decrypt(settings.smtp_password) : ''; - decryptedSettings = { ...settings, scaping_api, smtp_password }; + decryptedSettings = { + ...settings, + scaping_api, + smtp_password, + search_console_integrated: !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL), + }; } catch (error) { console.log('Error Decrypting Settings API Keys!'); } diff --git a/pages/domain/[slug]/index.tsx b/pages/domain/[slug]/index.tsx index 1061b43..186448f 100644 --- a/pages/domain/[slug]/index.tsx +++ b/pages/domain/[slug]/index.tsx @@ -29,19 +29,19 @@ const SingleDomain: NextPage = () => { const { data: domainsData } = useFetchDomains(router); const { keywordsData, keywordsLoading } = useFetchKeywords(router, setKeywordSPollInterval, keywordSPollInterval); - const theDomains: Domain[] = (domainsData && domainsData.domains) || []; + const theDomains: DomainType[] = (domainsData && domainsData.domains) || []; const theKeywords: KeywordType[] = keywordsData && keywordsData.keywords; - const activDomain: Domain|null = useMemo(() => { - let active:Domain|null = null; + const activDomain: DomainType|null = useMemo(() => { + let active:DomainType|null = null; if (domainsData?.domains && router.query?.slug) { - active = domainsData.domains.find((x:Domain) => x.slug === router.query.slug); + active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug); } return active; }, [router.query.slug, domainsData]); useEffect(() => { - console.log('appSettings.settings: ', appSettings && appSettings.settings); + // console.log('appSettings.settings: ', appSettings && appSettings.settings); if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) { setNoScrapprtError(true); } @@ -64,7 +64,7 @@ const SingleDomain: NextPage = () => { setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} />
    setShowAddDomain(true)} /> -
    +
    {activDomain && activDomain.domain && { showAddModal={setShowAddKeywords} showSettingsModal={setShowDomainSettings} exportCsv={() => exportCSV(theKeywords, activDomain.domain)} - />} + /> + }
    @@ -90,7 +92,6 @@ const SingleDomain: NextPage = () => { diff --git a/pages/domain/console/[slug]/index.tsx b/pages/domain/console/[slug]/index.tsx new file mode 100644 index 0000000..42bc732 --- /dev/null +++ b/pages/domain/console/[slug]/index.tsx @@ -0,0 +1,89 @@ +import React, { useMemo, useState } from 'react'; +import type { NextPage } from 'next'; +import Head from 'next/head'; +import { useRouter } from 'next/router'; +// import { useQuery } from 'react-query'; +// import toast from 'react-hot-toast'; +import { CSSTransition } from 'react-transition-group'; +import Sidebar from '../../../../components/common/Sidebar'; +import TopBar from '../../../../components/common/TopBar'; +import DomainHeader from '../../../../components/domains/DomainHeader'; +import AddDomain from '../../../../components/domains/AddDomain'; +import DomainSettings from '../../../../components/domains/DomainSettings'; +import exportCSV from '../../../../utils/exportcsv'; +import Settings from '../../../../components/settings/Settings'; +import { useFetchDomains } from '../../../../services/domains'; +import { useFetchSCKeywords } from '../../../../services/searchConsole'; +import SCKeywordsTable from '../../../../components/keywords/SCKeywordsTable'; +import { useFetchSettings } from '../../../../services/settings'; + +const DiscoverPage: NextPage = () => { + const router = useRouter(); + const [showDomainSettings, setShowDomainSettings] = useState(false); + const [showSettings, setShowSettings] = useState(false); + const [showAddDomain, setShowAddDomain] = useState(false); + const [scDateFilter, setSCDateFilter] = useState('thirtyDays'); + const { data: appSettings } = useFetchSettings(); + const { data: domainsData } = useFetchDomains(router); + const { data: keywordsData, isLoading: keywordsLoading, isFetching } = useFetchSCKeywords(router, !!(domainsData?.domains?.length)); + + const theDomains: DomainType[] = (domainsData && domainsData.domains) || []; + const theKeywords: SearchAnalyticsItem[] = keywordsData?.data && keywordsData.data[scDateFilter] ? keywordsData.data[scDateFilter] : []; + + const activDomain: DomainType|null = useMemo(() => { + let active:DomainType|null = null; + if (domainsData?.domains && router.query?.slug) { + active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug); + } + return active; + }, [router.query.slug, domainsData]); + + return ( +
    + {activDomain && activDomain.domain + && + {`${activDomain.domain} - SerpBear` } + + } + setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} /> +
    + setShowAddDomain(true)} /> +
    + {activDomain && activDomain.domain + && console.log('XXXXX')} + showSettingsModal={setShowDomainSettings} + exportCsv={() => exportCSV(theKeywords, activDomain.domain, scDateFilter)} + scFilter={scDateFilter} + setScFilter={(item:string) => setSCDateFilter(item)} + /> + } + +
    +
    + + + setShowAddDomain(false)} /> + + + + + + + setShowSettings(false)} /> + +
    + ); +}; + +export default DiscoverPage; diff --git a/pages/domain/insight/[slug]/index.tsx b/pages/domain/insight/[slug]/index.tsx new file mode 100644 index 0000000..b85221d --- /dev/null +++ b/pages/domain/insight/[slug]/index.tsx @@ -0,0 +1,89 @@ +import React, { useMemo, useState } from 'react'; +import type { NextPage } from 'next'; +import Head from 'next/head'; +import { useRouter } from 'next/router'; +// import { useQuery } from 'react-query'; +// import toast from 'react-hot-toast'; +import { CSSTransition } from 'react-transition-group'; +import Sidebar from '../../../../components/common/Sidebar'; +import TopBar from '../../../../components/common/TopBar'; +import DomainHeader from '../../../../components/domains/DomainHeader'; +import AddDomain from '../../../../components/domains/AddDomain'; +import DomainSettings from '../../../../components/domains/DomainSettings'; +import exportCSV from '../../../../utils/exportcsv'; +import Settings from '../../../../components/settings/Settings'; +import { useFetchDomains } from '../../../../services/domains'; +import { useFetchSCInsight } from '../../../../services/searchConsole'; +import SCInsight from '../../../../components/insight/Insight'; +import { useFetchSettings } from '../../../../services/settings'; + +const InsightPage: NextPage = () => { + const router = useRouter(); + const [showDomainSettings, setShowDomainSettings] = useState(false); + const [showSettings, setShowSettings] = useState(false); + const [showAddDomain, setShowAddDomain] = useState(false); + const [scDateFilter, setSCDateFilter] = useState('thirtyDays'); + const { data: appSettings } = useFetchSettings(); + const { data: domainsData } = useFetchDomains(router); + const { data: insightData } = useFetchSCInsight(router, !!(domainsData?.domains?.length)); + + const theDomains: DomainType[] = (domainsData && domainsData.domains) || []; + const theInsight: InsightDataType = insightData && insightData.data ? insightData.data : {}; + + const activDomain: DomainType|null = useMemo(() => { + let active:DomainType|null = null; + if (domainsData?.domains && router.query?.slug) { + active = domainsData.domains.find((x:DomainType) => x.slug === router.query.slug); + } + return active; + }, [router.query.slug, domainsData]); + + return ( +
    + {activDomain && activDomain.domain + && + {`${activDomain.domain} - SerpBear` } + + } + setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} /> +
    + setShowAddDomain(true)} /> +
    + {activDomain && activDomain.domain + && console.log('XXXXX')} + showSettingsModal={setShowDomainSettings} + exportCsv={() => exportCSV([], activDomain.domain, scDateFilter)} + scFilter={scDateFilter} + setScFilter={(item:string) => setSCDateFilter(item)} + /> + } + +
    +
    + + + setShowAddDomain(false)} /> + + + + + + + setShowSettings(false)} /> + +
    + ); +}; + +export default InsightPage; diff --git a/pages/domains/index.tsx b/pages/domains/index.tsx new file mode 100644 index 0000000..790b310 --- /dev/null +++ b/pages/domains/index.tsx @@ -0,0 +1,91 @@ +import React, { useEffect, useState } from 'react'; +import type { NextPage } from 'next'; +import Head from 'next/head'; +import { useRouter } from 'next/router'; +import { CSSTransition } from 'react-transition-group'; +import TopBar from '../../components/common/TopBar'; +import AddDomain from '../../components/domains/AddDomain'; +import Settings from '../../components/settings/Settings'; +import { useFetchSettings } from '../../services/settings'; +import { useFetchDomains } from '../../services/domains'; +import DomainItem from '../../components/domains/DomainItem'; +import Icon from '../../components/common/Icon'; + +const SingleDomain: NextPage = () => { + const router = useRouter(); + const [noScrapprtError, setNoScrapprtError] = useState(false); + const [showSettings, setShowSettings] = useState(false); + const [showAddDomain, setShowAddDomain] = useState(false); + const { data: appSettings } = useFetchSettings(); + const { data: domainsData, isLoading } = useFetchDomains(router, true); + + useEffect(() => { + console.log('Domains Data: ', domainsData); + }, [domainsData]); + + useEffect(() => { + // console.log('appSettings.settings: ', appSettings && appSettings.settings); + if (appSettings && appSettings.settings && (!appSettings.settings.scraper_type || (appSettings.settings.scraper_type === 'none'))) { + setNoScrapprtError(true); + } + }, [appSettings]); + + return ( +
    + {noScrapprtError && ( +
    + A Scrapper/Proxy has not been set up Yet. Open Settings to set it up and start using the app. +
    + )} + + {'Domains - SerpBear' } + + setShowSettings(true)} showAddModal={() => setShowAddDomain(true)} /> + +
    +
    +
    {domainsData?.domains?.length || 0} Domains
    +
    + +
    +
    +
    + {domainsData?.domains && domainsData.domains.map((domain:DomainType) => { + return ; + })} + {isLoading && ( +
    + Loading Domains... +
    + )} + {!isLoading && domainsData && domainsData.domains && domainsData.domains.length === 0 && ( +
    + No Domains Found. Add a Domain to get started! +
    + )} +
    +
    + + + setShowAddDomain(false)} /> + + + setShowSettings(false)} /> + +
    + ); +}; + +export default SingleDomain; diff --git a/pages/index.tsx b/pages/index.tsx index 9012398..22ade90 100644 --- a/pages/index.tsx +++ b/pages/index.tsx @@ -1,37 +1,14 @@ import type { NextPage } from 'next'; -import { useEffect, useState } from 'react'; +import { useEffect } from 'react'; import Head from 'next/head'; import { useRouter } from 'next/router'; import { Toaster } from 'react-hot-toast'; import Icon from '../components/common/Icon'; -import AddDomain from '../components/domains/AddDomain'; const Home: NextPage = () => { - const [loading, setLoading] = useState(false); - const [domains, setDomains] = useState([]); const router = useRouter(); useEffect(() => { - setLoading(true); - fetch(`${window.location.origin}/api/domains`) - .then((result) => { - if (result.status === 401) { - router.push('/login'); - } - return result.json(); - }) - .then((domainsRes:any) => { - if (domainsRes?.domains && domainsRes.domains.length > 0) { - const firstDomainItem = domainsRes.domains[0].slug; - setDomains(domainsRes.domains); - router.push(`/domain/${firstDomainItem}`); - } - setLoading(false); - return false; - }) - .catch((err) => { - console.log(err); - setLoading(false); - }); + router.push('/domains'); }, [router]); return ( @@ -46,7 +23,6 @@ const Home: NextPage = () => { - {!loading && domains.length === 0 && console.log('Cannot Close Modal!')} />}
    ); }; diff --git a/services/domains.tsx b/services/domains.tsx index 58521dc..a722ee3 100644 --- a/services/domains.tsx +++ b/services/domains.tsx @@ -4,11 +4,11 @@ import { useMutation, useQuery, useQueryClient } from 'react-query'; type UpdatePayload = { domainSettings: DomainSettings, - domain: Domain + domain: DomainType } -export async function fetchDomains(router: NextRouter) { - const res = await fetch(`${window.location.origin}/api/domains`, { method: 'GET' }); +export async function fetchDomains(router: NextRouter, withStats:boolean) { + const res = await fetch(`${window.location.origin}/api/domains${withStats ? '?withstats=true' : ''}`, { method: 'GET' }); if (res.status >= 400 && res.status < 600) { if (res.status === 401) { console.log('Unauthorized!!'); @@ -19,8 +19,8 @@ export async function fetchDomains(router: NextRouter) { return res.json(); } -export function useFetchDomains(router: NextRouter) { - return useQuery('domains', () => fetchDomains(router)); +export function useFetchDomains(router: NextRouter, withStats:boolean = false) { + return useQuery('domains', () => fetchDomains(router, withStats)); } export function useAddDomain(onSuccess:Function) { @@ -37,7 +37,7 @@ export function useAddDomain(onSuccess:Function) { }, { onSuccess: async (data) => { console.log('Domain Added!!!', data); - const newDomain:Domain = data.domain; + const newDomain:DomainType = data.domain; toast(`${newDomain.domain} Added Successfully!`, { icon: '✔️' }); onSuccess(false); if (newDomain && newDomain.slug) { @@ -78,7 +78,7 @@ export function useUpdateDomain(onSuccess:Function) { export function useDeleteDomain(onSuccess:Function) { const queryClient = useQueryClient(); - return useMutation(async (domain:Domain) => { + return useMutation(async (domain:DomainType) => { const res = await fetch(`${window.location.origin}/api/domains?domain=${domain.domain}`, { method: 'DELETE' }); if (res.status >= 400 && res.status < 600) { throw new Error('Bad response from server'); diff --git a/services/keywords.tsx b/services/keywords.tsx index 71bd6b6..828cde4 100644 --- a/services/keywords.tsx +++ b/services/keywords.tsx @@ -2,14 +2,6 @@ import toast from 'react-hot-toast'; import { NextRouter } from 'next/router'; import { useMutation, useQuery, useQueryClient } from 'react-query'; -type KeywordsInput = { - keywords: string, - device: string, - country: string, - domain: string, - tags: string, -} - export const fetchKeywords = async (router: NextRouter) => { if (!router.query.slug) { return []; } const res = await fetch(`${window.location.origin}/api/keywords?domain=${router.query.slug}`, { method: 'GET' }); @@ -44,9 +36,9 @@ export function useFetchKeywords(router: NextRouter, setKeywordSPollInterval:Fun export function useAddKeywords(onSuccess:Function) { const queryClient = useQueryClient(); - return useMutation(async (newKeywords:KeywordsInput) => { + return useMutation(async (keywords:KeywordAddPayload[]) => { const headers = new Headers({ 'Content-Type': 'application/json', Accept: 'application/json' }); - const fetchOpts = { method: 'POST', headers, body: JSON.stringify(newKeywords) }; + const fetchOpts = { method: 'POST', headers, body: JSON.stringify({ keywords }) }; const res = await fetch(`${window.location.origin}/api/keywords`, fetchOpts); if (res.status >= 400 && res.status < 600) { throw new Error('Bad response from server'); diff --git a/services/searchConsole.ts b/services/searchConsole.ts new file mode 100644 index 0000000..f269583 --- /dev/null +++ b/services/searchConsole.ts @@ -0,0 +1,38 @@ +import { NextRouter } from 'next/router'; +import { useQuery } from 'react-query'; + +export async function fetchSCKeywords(router: NextRouter) { + // if (!router.query.slug) { throw new Error('Invalid Domain Name'); } + const res = await fetch(`${window.location.origin}/api/searchconsole?domain=${router.query.slug}`, { method: 'GET' }); + if (res.status >= 400 && res.status < 600) { + if (res.status === 401) { + console.log('Unauthorized!!'); + router.push('/login'); + } + throw new Error('Bad response from server'); + } + return res.json(); +} + +export function useFetchSCKeywords(router: NextRouter, domainLoaded: boolean = false) { + // console.log('ROUTER: ', router); + return useQuery('sckeywords', () => router.query.slug && fetchSCKeywords(router), { enabled: domainLoaded }); +} + +export async function fetchSCInsight(router: NextRouter) { + // if (!router.query.slug) { throw new Error('Invalid Domain Name'); } + const res = await fetch(`${window.location.origin}/api/insight?domain=${router.query.slug}`, { method: 'GET' }); + if (res.status >= 400 && res.status < 600) { + if (res.status === 401) { + console.log('Unauthorized!!'); + router.push('/login'); + } + throw new Error('Bad response from server'); + } + return res.json(); +} + +export function useFetchSCInsight(router: NextRouter, domainLoaded: boolean = false) { + // console.log('ROUTER: ', router); + return useQuery('scinsight', () => router.query.slug && fetchSCInsight(router), { enabled: domainLoaded }); +} diff --git a/styles/globals.css b/styles/globals.css index 29def16..e973961 100644 --- a/styles/globals.css +++ b/styles/globals.css @@ -9,7 +9,7 @@ body { } .domKeywords { - min-height: 70vh; + /* min-height: 70vh; */ border-color: #e9ebff; box-shadow: 0 0 20px rgb(20 34 71 / 5%); } @@ -83,21 +83,31 @@ body { } .domKeywords_head--alpha_desc .domKeywords_head_keyword::after, -.domKeywords_head--pos_desc .domKeywords_head_position::after { +.domKeywords_head--pos_desc .domKeywords_head_position::after, +.domKeywords_head--imp_desc .domKeywords_head_imp::after, +.domKeywords_head--visits_desc .domKeywords_head_visits::after, +.domKeywords_head--ctr_desc .domKeywords_head_ctr::after { content: "↓"; display: inline-block; - margin-left: 2px; + margin-left: 4px; font-size: 14px; opacity: 0.8; + font-family: sans-serif; + font-weight: bold; } .domKeywords_head--alpha_asc .domKeywords_head_keyword::after, -.domKeywords_head--pos_asc .domKeywords_head_position::after { +.domKeywords_head--pos_asc .domKeywords_head_position::after, +.domKeywords_head--imp_asc .domKeywords_head_imp::after, +.domKeywords_head--visits_asc .domKeywords_head_visits::after, +.domKeywords_head--ctr_asc .domKeywords_head_ctr::after { content: "↑"; display: inline-block; - margin-left: 2px; + margin-left: 4px; font-size: 14px; opacity: 0.8; + font-family: sans-serif; + font-weight: bold; } .keywordDetails__section__results { @@ -168,6 +178,64 @@ body { transition: all 300ms; } +.domItem { + transition: all 0.15s linear; + border-color: #e9ebff; + box-shadow: 0 0 20px rgb(20 34 71 / 5%); +} + +.domItem:hover h3 { + color: #364aff +} + +.noDomains { + border-color: #e9ebff; + box-shadow: 0 0 20px rgb(20 34 71 / 5%); +} + +.domItem:hover { + border-color: #9499d8; + box-shadow: 0 0 20px rgb(30 65 161 / 25%); +} + +.domain_selector .selected, +.insight_selector .selected { + width: 100%; +} + +.domain_selector .select, +.insight_selector .select { + position: relative; +} + +.domain_selector .select_list, +.insight_selector .select_list { + width: 100%; +} + +.domain_selector .selected > span:nth-child(1), +.insight_selector .selected > span:nth-child(1) { + width: 100% !important; +} + +.dom_sc_stats > div::after, +.dom_stats > div::after { + content: ""; + width: 1px; + height: 35px; + background: #eee; + position: absolute; + right: 0; +} + +.dom_sc_stats > div:nth-child(3)::after { + display: none; +} + +.dom_stats > div:nth-child(2)::after { + display: none; +} + @media (min-width: 1024px) { /* Domain Header Button Tooltips */ .domheader_action_button:hover i { @@ -178,15 +246,16 @@ body { .domheader_action_button i { display: block; position: absolute; - width: 100px; left: -40px; top: -22px; background: #222; border-radius: 3px; color: #fff; font-size: 12px; + padding: 0 10px; padding-bottom: 3px; transition: all 0.2s linear; + width: max-content; } .domheader_action_button i::after { @@ -202,4 +271,14 @@ body { right: 0; margin: 0 auto; } + + .domkeywordsTable--keywords.domkeywordsTable--hasSC .domKeywords_keywords::before { + content: ""; + position: absolute; + width: 1px; + height: 100%; + background: #eff0f1; + top: 0; + right: 240px; + } } diff --git a/types.d.ts b/types.d.ts index 99e5b1a..a4cab09 100644 --- a/types.d.ts +++ b/types.d.ts @@ -1,15 +1,20 @@ /* eslint-disable no-unused-vars */ -type Domain = { +type DomainType = { ID: number, domain: string, slug: string, - tags?: string[], + tags?: string, notification: boolean, notification_interval: string, notification_emails: string, lastUpdated: string, added: string, - keywordCount: number + keywordCount?: number, + keywordsUpdated?: string, + avgPosition?: number, + scVisits?: number, + scImpressions?: number, + scPosition?: number, } type KeywordHistory = { @@ -31,7 +36,9 @@ type KeywordType = { url: string, tags: string[], updating: boolean, - lastUpdateError: {date: string, error: string, scraper: string} | false + lastUpdateError: {date: string, error: string, scraper: string} | false, + scData?: KeywordSCData, + uid?: string } type KeywordLastResult = { @@ -50,6 +57,10 @@ type countryData = { [ISO:string] : string[] } +type countryCodeData = { + [ISO:string] : string +} + type DomainSettings = { notification_interval: string, notification_emails: string, @@ -65,5 +76,90 @@ type SettingsType = { smtp_server: string, smtp_port: string, smtp_username: string, - smtp_password: string + smtp_password: string, + search_console_integrated?: boolean, +} + +type KeywordSCDataChild = { + yesterday: number, + threeDays: number, + sevenDays: number, + thirtyDays: number, + avgSevenDays: number, + avgThreeDays: number, + avgThirtyDays: number, +} +type KeywordSCData = { + impressions: KeywordSCDataChild, + visits: KeywordSCDataChild, + ctr: KeywordSCDataChild, + position:KeywordSCDataChild +} + +type KeywordAddPayload = { + keyword: string, + device: string, + country: string, + domain: string, + tags: string, +} + +type SearchAnalyticsRawItem = { + keys: string[], + clicks: number, + impressions: number, + ctr: number, + position: number, } + +type SearchAnalyticsStat = { + date: string, + clicks: number, + impressions: number, + ctr: number, + position: number, +} + +type InsightDataType = { + stats: SearchAnalyticsStat[]|null, + keywords: SCInsightItem[], + countries: SCInsightItem[], + pages: SCInsightItem[], +} + +type SCInsightItem = { + clicks: number, + impressions: number, + ctr: number, + position: number, + countries?: number, + country?: string, + keyword?: string, + keywords?: number, + page?: string, + date?: string +} + +type SearchAnalyticsItem = { + keyword: string, + uid: string, + device: string, + page: string, + country: string, + clicks: number, + impressions: number, + ctr: number, + position: number, + date?: string +} + +type SCDomainDataType = { + threeDays : SearchAnalyticsItem[], + sevenDays : SearchAnalyticsItem[], + thirtyDays : SearchAnalyticsItem[], + lastFetched?: string, + lastFetchError?: string, + stats? : SearchAnalyticsStat[], +} + +type SCKeywordType = SearchAnalyticsItem; diff --git a/utils/SCsortFilter.ts b/utils/SCsortFilter.ts new file mode 100644 index 0000000..1e2c444 --- /dev/null +++ b/utils/SCsortFilter.ts @@ -0,0 +1,82 @@ +/** + * Sorrt Keywords by user's given input. + * @param {SCKeywordType[]} theKeywords - The Keywords to sort. + * @param {string} sortBy - The sort method. + * @returns {SCKeywordType[]} + */ +export const SCsortKeywords = (theKeywords:SCKeywordType[], sortBy:string) : SCKeywordType[] => { + let sortedItems = []; + const keywords = theKeywords.map((k) => ({ ...k, position: k.position === 0 ? 111 : k.position })); + switch (sortBy) { + case 'imp_asc': + sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.impressions - a.impressions); + break; + case 'imp_desc': + sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.impressions - b.impressions); + break; + case 'visits_asc': + sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.clicks - a.clicks); + break; + case 'visits_desc': + sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.clicks - b.clicks); + break; + case 'ctr_asc': + sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => b.ctr - a.ctr); + break; + case 'ctr_desc': + sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => a.ctr - b.ctr); + break; + case 'pos_asc': + sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.position > a.position ? 1 : -1)); + sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position })); + break; + case 'pos_desc': + sortedItems = keywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.position > b.position ? 1 : -1)); + sortedItems = sortedItems.map((k) => ({ ...k, position: k.position === 111 ? 0 : k.position })); + break; + case 'alpha_asc': + sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (b.keyword > a.keyword ? 1 : -1)); + break; + case 'alpha_desc': + sortedItems = theKeywords.sort((a: SCKeywordType, b: SCKeywordType) => (a.keyword > b.keyword ? 1 : -1)); + break; + default: + return theKeywords; + } + + return sortedItems; +}; + +/** + * Filters the Keywords by Device when the Device buttons are switched + * @param {SCKeywordType[]} sortedKeywords - The Sorted Keywords. + * @param {string} device - Device name (desktop or mobile). + * @returns {{desktop: SCKeywordType[], mobile: SCKeywordType[] } } + */ +export const SCkeywordsByDevice = (sortedKeywords: SCKeywordType[], device: string): {[key: string]: SCKeywordType[] } => { + const deviceKeywords: {[key:string] : SCKeywordType[]} = { desktop: [], mobile: [] }; + sortedKeywords.forEach((keyword) => { + if (keyword.device === device) { deviceKeywords[device].push(keyword); } + }); + return deviceKeywords; +}; + +/** + * Filters the keywords by country, search string or tags. + * @param {SCKeywordType[]} keywords - The keywords. + * @param {KeywordFilters} filterParams - The user Selected filter object. + * @returns {SCKeywordType[]} + */ +export const SCfilterKeywords = (keywords: SCKeywordType[], filterParams: KeywordFilters):SCKeywordType[] => { + const filteredItems:SCKeywordType[] = []; + keywords.forEach((keywrd) => { + const countryMatch = filterParams.countries.length === 0 ? true : filterParams.countries && filterParams.countries.includes(keywrd.country); + const searchMatch = !filterParams.search ? true : filterParams.search && keywrd.keyword.includes(filterParams.search); + + if (countryMatch && searchMatch) { + filteredItems.push(keywrd); + } + }); + + return filteredItems; +}; diff --git a/utils/countries.ts b/utils/countries.ts index c11a1da..3a1fae4 100644 --- a/utils/countries.ts +++ b/utils/countries.ts @@ -253,6 +253,262 @@ const countries: countryData = { ZA: ['South Africa', 'Pretoria', 'af'], ZM: ['Zambia', 'Lusaka', 'en'], ZW: ['Zimbabwe', 'Harare', 'en'], + ZZ: ['Unknown', 'Unknown', 'en'], }; +export const countryAlphaTwoCodes: countryCodeData = { + AFG: 'AF', + ALA: 'AX', + ALB: 'AL', + DZA: 'DZ', + ASM: 'AS', + AND: 'AD', + AGO: 'AO', + AIA: 'AI', + ATA: 'AQ', + ATG: 'AG', + ARG: 'AR', + ARM: 'AM', + ABW: 'AW', + AUS: 'AU', + AUT: 'AT', + AZE: 'AZ', + BHS: 'BS', + BHR: 'BH', + BGD: 'BD', + BRB: 'BB', + BLR: 'BY', + BEL: 'BE', + BLZ: 'BZ', + BEN: 'BJ', + BMU: 'BM', + BTN: 'BT', + BOL: 'BO', + BES: 'BQ', + BIH: 'BA', + BWA: 'BW', + BVT: 'BV', + BRA: 'BR', + IOT: 'IO', + BRN: 'BN', + BGR: 'BG', + BFA: 'BF', + BDI: 'BI', + CPV: 'CV', + KHM: 'KH', + CMR: 'CM', + CAN: 'CA', + CYM: 'KY', + CAF: 'CF', + TCD: 'TD', + CHL: 'CL', + CHN: 'CN', + CXR: 'CX', + CCK: 'CC', + COL: 'CO', + COM: 'KM', + COG: 'CG', + COD: 'CD', + COK: 'CK', + CRI: 'CR', + CIV: 'CI', + HRV: 'HR', + CUB: 'CU', + CUW: 'CW', + CYP: 'CY', + CZE: 'CZ', + DNK: 'DK', + DJI: 'DJ', + DMA: 'DM', + DOM: 'DO', + ECU: 'EC', + EGY: 'EG', + SLV: 'SV', + GNQ: 'GQ', + ERI: 'ER', + EST: 'EE', + SWZ: 'SZ', + ETH: 'ET', + FLK: 'FK', + FRO: 'FO', + FJI: 'FJ', + FIN: 'FI', + FRA: 'FR', + GUF: 'GF', + PYF: 'PF', + ATF: 'TF', + GAB: 'GA', + GMB: 'GM', + GEO: 'GE', + DEU: 'DE', + GHA: 'GH', + GIB: 'GI', + GRC: 'GR', + GRL: 'GL', + GRD: 'GD', + GLP: 'GP', + GUM: 'GU', + GTM: 'GT', + GGY: 'GG', + GIN: 'GN', + GNB: 'GW', + GUY: 'GY', + HTI: 'HT', + HMD: 'HM', + VAT: 'VA', + HND: 'HN', + HKG: 'HK', + HUN: 'HU', + ISL: 'IS', + IND: 'IN', + IDN: 'ID', + IRN: 'IR', + IRQ: 'IQ', + IRL: 'IE', + IMN: 'IM', + ISR: 'IL', + ITA: 'IT', + JAM: 'JM', + JPN: 'JP', + JEY: 'JE', + JOR: 'JO', + KAZ: 'KZ', + KEN: 'KE', + KIR: 'KI', + PRK: 'KP', + KOR: 'KR', + KWT: 'KW', + KGZ: 'KG', + LAO: 'LA', + LVA: 'LV', + LBN: 'LB', + LSO: 'LS', + LBR: 'LR', + LBY: 'LY', + LIE: 'LI', + LTU: 'LT', + LUX: 'LU', + MAC: 'MO', + MDG: 'MG', + MWI: 'MW', + MYS: 'MY', + MDV: 'MV', + MLI: 'ML', + MLT: 'MT', + MHL: 'MH', + MTQ: 'MQ', + MRT: 'MR', + MUS: 'MU', + MYT: 'YT', + MEX: 'MX', + FSM: 'FM', + MDA: 'MD', + MCO: 'MC', + MNG: 'MN', + MNE: 'ME', + MSR: 'MS', + MAR: 'MA', + MOZ: 'MZ', + MMR: 'MM', + NAM: 'NA', + NRU: 'NR', + NPL: 'NP', + NLD: 'NL', + NCL: 'NC', + NZL: 'NZ', + NIC: 'NI', + NER: 'NE', + NGA: 'NG', + NIU: 'NU', + NFK: 'NF', + MKD: 'MK', + MNP: 'MP', + NOR: 'NO', + OMN: 'OM', + PAK: 'PK', + PLW: 'PW', + PSE: 'PS', + PAN: 'PA', + PNG: 'PG', + PRY: 'PY', + PER: 'PE', + PHL: 'PH', + PCN: 'PN', + POL: 'PL', + PRT: 'PT', + PRI: 'PR', + QAT: 'QA', + REU: 'RE', + ROU: 'RO', + RUS: 'RU', + RWA: 'RW', + BLM: 'BL', + SHN: 'SH', + KNA: 'KN', + LCA: 'LC', + MAF: 'MF', + SPM: 'PM', + VCT: 'VC', + WSM: 'WS', + SMR: 'SM', + STP: 'ST', + SAU: 'SA', + SEN: 'SN', + SRB: 'RS', + SYC: 'SC', + SLE: 'SL', + SGP: 'SG', + SXM: 'SX', + SVK: 'SK', + SVN: 'SI', + SLB: 'SB', + SOM: 'SO', + ZAF: 'ZA', + SGS: 'GS', + SSD: 'SS', + ESP: 'ES', + LKA: 'LK', + SDN: 'SD', + SUR: 'SR', + SJM: 'SJ', + SWE: 'SE', + CHE: 'CH', + SYR: 'SY', + TWN: 'TW', + TJK: 'TJ', + TZA: 'TZ', + THA: 'TH', + TLS: 'TL', + TGO: 'TG', + TKL: 'TK', + TON: 'TO', + TTO: 'TT', + TUN: 'TN', + TUR: 'TR', + TKM: 'TM', + TCA: 'TC', + TUV: 'TV', + UGA: 'UG', + UKR: 'UA', + ARE: 'AE', + GBR: 'GB', + USA: 'US', + UMI: 'UM', + URY: 'UY', + UZB: 'UZ', + VUT: 'VU', + VEN: 'VE', + VNM: 'VN', + VGB: 'VG', + VIR: 'VI', + WLF: 'WF', + ESH: 'EH', + YEM: 'YE', + ZMB: 'ZM', + ZWE: 'ZW', + ZZZ: 'ZZ', +}; + +export const getCountryCodeFromAlphaThree = (AlphaThreeCode:string): string => countryAlphaTwoCodes[AlphaThreeCode]; + export default countries; diff --git a/utils/domains.ts b/utils/domains.ts new file mode 100644 index 0000000..f200e66 --- /dev/null +++ b/utils/domains.ts @@ -0,0 +1,45 @@ +import Keyword from '../database/models/keyword'; +import parseKeywords from './parseKeywords'; +import { readLocalSCData } from './searchConsole'; + +const getdomainStats = async (domains:DomainType[]): Promise => { + const finalDomains: DomainType[] = []; + console.log('domains: ', domains.length); + + for (const domain of domains) { + const domainWithStat = domain; + + // First Get ALl The Keywords for this Domain + const allKeywords:Keyword[] = await Keyword.findAll({ where: { domain: domain.domain } }); + const keywords: KeywordType[] = parseKeywords(allKeywords.map((e) => e.get({ plain: true }))); + domainWithStat.keywordCount = keywords.length; + const keywordPositions = keywords.reduce((acc, itm) => (acc + itm.position), 0); + const KeywordsUpdateDates: number[] = keywords.reduce((acc: number[], itm) => [...acc, new Date(itm.lastUpdated).getTime()], [0]); + domainWithStat.keywordsUpdated = new Date(Math.max(...KeywordsUpdateDates)).toJSON(); + domainWithStat.avgPosition = Math.round(keywordPositions / keywords.length); + + // Then Load the SC File and read the stats and calculate the Last 7 days stats + const localSCData = await readLocalSCData(domain.domain); + const days = 7; + if (localSCData && localSCData.stats && localSCData.stats.length) { + const lastSevenStats = localSCData.stats.slice(-days); + const totalStats = lastSevenStats.reduce((acc, item) => { + return { + impressions: item.impressions + acc.impressions, + clicks: item.clicks + acc.clicks, + ctr: item.ctr + acc.ctr, + position: item.position + acc.position, + }; + }, { impressions: 0, clicks: 0, ctr: 0, position: 0 }); + domainWithStat.scVisits = totalStats.clicks; + domainWithStat.scImpressions = totalStats.impressions; + domainWithStat.scPosition = Math.round(totalStats.position / days); + } + + finalDomains.push(domainWithStat); + } + + return finalDomains; +}; + +export default getdomainStats; diff --git a/utils/exportcsv.ts b/utils/exportcsv.ts index 78e01c1..f1a9768 100644 --- a/utils/exportcsv.ts +++ b/utils/exportcsv.ts @@ -6,21 +6,36 @@ import countries from './countries'; * @param {string} domain - The domain name. * @returns {void} */ -const exportCSV = (keywords: KeywordType[], domain:string) => { - const csvHeader = 'ID,Keyword,Position,URL,Country,Device,Updated,Added,Tags\r\n'; +const exportCSV = (keywords: KeywordType[] | SCKeywordType[], domain:string, scDataDuration = 'lastThreeDays') => { + const isSCKeywords = !!(keywords && keywords[0] && keywords[0].uid); + let csvHeader = 'ID,Keyword,Position,URL,Country,Device,Updated,Added,Tags\r\n'; let csvBody = ''; + let fileName = `${domain}-keywords_serp.csv`; - keywords.forEach((keywordData) => { - const { ID, keyword, position, url, country, device, lastUpdated, added, tags } = keywordData; - // eslint-disable-next-line max-len - csvBody += `${ID}, ${keyword}, ${position === 0 ? '-' : position}, ${url || '-'}, ${countries[country][0]}, ${device}, ${lastUpdated}, ${added}, ${tags.join(',')}\r\n`; - }); + console.log(keywords[0]); + console.log('isSCKeywords:', isSCKeywords); + + if (isSCKeywords) { + csvHeader = 'ID,Keyword,Position,Impressions,Clicks,CTR,Country,Device\r\n'; + fileName = `${domain}-search-console-${scDataDuration}.csv`; + keywords.forEach((keywordData, index) => { + const { keyword, position, country, device, clicks, impressions, ctr } = keywordData as SCKeywordType; + // eslint-disable-next-line max-len + csvBody += `${index}, ${keyword}, ${position === 0 ? '-' : position}, ${impressions}, ${clicks}, ${ctr}, ${countries[country][0]}, ${device}\r\n`; + }); + } else { + keywords.forEach((keywordData) => { + const { ID, keyword, position, url, country, device, lastUpdated, added, tags } = keywordData as KeywordType; + // eslint-disable-next-line max-len + csvBody += `${ID}, ${keyword}, ${position === 0 ? '-' : position}, ${url || '-'}, ${countries[country][0]}, ${device}, ${lastUpdated}, ${added}, ${tags.join(',')}\r\n`; + }); + } const blob = new Blob([csvHeader + csvBody], { type: 'text/csv;charset=utf-8;' }); const url = URL.createObjectURL(blob); const link = document.createElement('a'); link.setAttribute('href', url); - link.setAttribute('download', `${domain}-keywords_serp.csv`); + link.setAttribute('download', fileName); link.style.visibility = 'hidden'; document.body.appendChild(link); link.click(); diff --git a/utils/generateEmail.ts b/utils/generateEmail.ts index 1e833f5..ec03cf1 100644 --- a/utils/generateEmail.ts +++ b/utils/generateEmail.ts @@ -1,13 +1,25 @@ import dayjs from 'dayjs'; import { readFile } from 'fs/promises'; import path from 'path'; - -const serpBearLogo = 'https://i.imgur.com/ikAdjQq.png'; -const mobileIcon = 'https://i.imgur.com/SqXD9rd.png'; -const desktopIcon = 'https://i.imgur.com/Dx3u0XD.png'; +import { getKeywordsInsight, getPagesInsight } from './insight'; +import { readLocalSCData } from './searchConsole'; + +const serpBearLogo = 'https://erevanto.sirv.com/Images/serpbear/ikAdjQq.png'; +const mobileIcon = 'https://erevanto.sirv.com/Images/serpbear/SqXD9rd.png'; +const desktopIcon = 'https://erevanto.sirv.com/Images/serpbear/Dx3u0XD.png'; +const googleIcon = 'https://erevanto.sirv.com/Images/serpbear/Sx3u0X9.png'; + +type SCStatsObject = { + [key:string]: { + html: string, + label: string, + clicks?: number, + impressions?: number + }, +} /** - * Geenrate Human readable Time string. + * Generate Human readable Time string. * @param {number} date - Keywords to scrape * @returns {string} */ @@ -98,7 +110,113 @@ const generateEmail = async (domainName:string, keywords:KeywordType[]) : Promis .replace('{{stat}}', stat) .replace('{{preheader}}', stat); - return updatedEmail; + const isConsoleIntegrated = !!(process.env.SEARCH_CONSOLE_PRIVATE_KEY && process.env.SEARCH_CONSOLE_CLIENT_EMAIL); + const htmlWithSCStats = isConsoleIntegrated ? await generateGoogeleConsoleStats(domainName) : ''; + const emailHTML = updatedEmail.replace('{{SCStatsTable}}', htmlWithSCStats); + + // await writeFile('testemail.html', emailHTML, { encoding: 'utf-8' }); + + return emailHTML; +}; + +/** + * Generate the Email HTML for Google Search Console Data. + * @param {string} domainName - The Domain name for which to generate the HTML. + * @returns {Promise} + */ +const generateGoogeleConsoleStats = async (domainName:string): Promise => { + if (!domainName) return ''; + + const localSCData = await readLocalSCData(domainName); + if (!localSCData || !localSCData.stats || !localSCData.stats.length) { + return ''; // IF No SC Data Found, Abot the process. + } + + const scData:SCStatsObject = { + stats: { html: '', label: 'Performance for Last 7 Days', clicks: 0, impressions: 0 }, + keywords: { html: '', label: 'Top 5 Keywords' }, + pages: { html: '', label: 'Top 5 Pages' }, + }; + const SCStats = localSCData && localSCData.stats && Array.isArray(localSCData.stats) ? localSCData.stats.reverse().slice(0, 7) : []; + const keywords = getKeywordsInsight(localSCData, 'clicks', 'sevenDays'); + const pages = getPagesInsight(localSCData, 'clicks', 'sevenDays'); + const genColumn = (item:SCInsightItem, firstColumKey:string):string => { + return ` + ${item[firstColumKey as keyof SCInsightItem]} + ${item.clicks} + ${item.impressions} + ${Math.round(item.position)} + `; + }; + if (SCStats.length > 0) { + scData.stats.html = SCStats.reduce((acc, item) => acc + genColumn(item, 'date'), ''); + } + if (keywords.length > 0) { + scData.keywords.html = keywords.slice(0, 5).reduce((acc, item) => acc + genColumn(item, 'keyword'), ''); + } + if (pages.length > 0) { + scData.pages.html = pages.slice(0, 5).reduce((acc, item) => acc + genColumn(item, 'page'), ''); + } + scData.stats.clicks = SCStats.reduce((acc, item) => acc + item.clicks, 0); + scData.stats.impressions = SCStats.reduce((acc, item) => acc + item.impressions, 0); + + // Create Stats Start, End Date + const months = ['Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 'Jul', 'Aug', 'Sep', 'Oct', 'Nov', 'Dec']; + const endDate = new Date(SCStats[0].date); + const startDate = new Date(SCStats[SCStats.length - 1].date); + + // Add the SC header Title + let htmlWithSCStats = ` + + + + + + `; + + // Add the SC Data Tables + Object.keys(scData).forEach((itemKey) => { + const scItem = scData[itemKey as keyof SCStatsObject]; + const scItemFirstColName = itemKey === 'stats' ? 'Date' : `${itemKey[0].toUpperCase()}${itemKey.slice(1)}`; + htmlWithSCStats += ` + + + ${scItem.clicks && scItem.impressions ? ( + `` + ) + : '' + } + + + + + + + + + `; + }); + + return htmlWithSCStats; }; export default generateEmail; diff --git a/utils/insight.ts b/utils/insight.ts new file mode 100644 index 0000000..148f931 --- /dev/null +++ b/utils/insight.ts @@ -0,0 +1,142 @@ +export const sortInsightItems = (items:SCInsightItem[], sortBy: string = 'clicks') => { + const sortKey = sortBy as keyof SCInsightItem; + let sortedItems = []; + switch (sortKey) { + case 'clicks': + sortedItems = items.sort((a, b) => (b.clicks > a.clicks ? 1 : -1)); + break; + case 'impressions': + sortedItems = items.sort((a, b) => (b.impressions > a.impressions ? 1 : -1)); + break; + case 'position': + sortedItems = items.sort((a, b) => (b.position > a.position ? 1 : -1)); + break; + default: + sortedItems = items; + break; + } + return sortedItems; +}; + +export const getCountryInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => { + const keywordsCounts: { [key:string]: string[] } = {}; + const countryItems: { [key:string]: SCInsightItem } = {}; + const dateKey = queryDate as keyof SCDomainDataType; + const scData = SCData[dateKey] ? SCData[dateKey] as SearchAnalyticsItem[] : []; + const allCountries: string[] = [...new Set(scData.map((item) => item.country))]; + + allCountries.forEach((countryKey:string) => { + const itemData = { clicks: 0, impressions: 0, ctr: 0, position: 0 }; + scData.forEach((itm) => { + if (itm.country === countryKey) { + itemData.clicks += itm.clicks; + itemData.impressions += itm.impressions; + itemData.ctr += itm.ctr; + itemData.position += itm.position; + if (!keywordsCounts[itm.country]) { + keywordsCounts[itm.country] = []; + } + if (keywordsCounts[itm.country] && !keywordsCounts[itm.country].includes(itm.keyword)) { + keywordsCounts[itm.country].push(itm.keyword); + } + } + }); + countryItems[countryKey] = itemData; + }); + + const countryInsight: SCInsightItem[] = Object.keys(countryItems).map((countryCode:string) => { + return { + ...countryItems[countryCode], + position: Math.round(countryItems[countryCode].position / keywordsCounts[countryCode].length), + ctr: countryItems[countryCode].ctr / keywordsCounts[countryCode].length, + keywords: keywordsCounts[countryCode].length, + country: countryCode, + }; + }); + + return sortBy ? sortInsightItems(countryInsight, sortBy) : countryInsight; +}; + +export const getKeywordsInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => { + const keywordItems: { [key:string]: SCInsightItem } = {}; + const keywordCounts: { [key:string]: number } = {}; + const countriesCount: { [key:string]: string[] } = {}; + const dateKey = queryDate as keyof SCDomainDataType; + const scData = SCData[dateKey] ? SCData[dateKey] as SearchAnalyticsItem[] : []; + const allKeywords: string[] = [...new Set(scData.map((item) => item.keyword))]; + + allKeywords.forEach((keyword:string) => { + const itemData = { clicks: 0, impressions: 0, ctr: 0, position: 0 }; + const keywordKey = keyword.replaceAll(' ', '_'); + scData.forEach((itm) => { + if (itm.keyword === keyword) { + itemData.clicks += itm.clicks; + itemData.impressions += itm.impressions; + itemData.ctr += itm.ctr; + itemData.position += itm.position; + if (!countriesCount[keywordKey]) { + countriesCount[keywordKey] = []; + } + if (countriesCount[keywordKey] && !countriesCount[keywordKey].includes(itm.country)) { + countriesCount[keywordKey].push(itm.keyword); + } + keywordCounts[keywordKey] = keywordCounts[keywordKey] ? keywordCounts[keywordKey] + 1 : 1; + } + }); + keywordItems[keywordKey] = itemData; + }); + + const keywordInsight: SCInsightItem[] = Object.keys(keywordItems).map((keyword:string) => { + return { + ...keywordItems[keyword], + position: Math.round(keywordItems[keyword].position / keywordCounts[keyword]), + ctr: keywordItems[keyword].ctr / keywordCounts[keyword], + countries: countriesCount[keyword].length, + keyword: keyword.replaceAll('_', ' '), + }; + }); + + return sortBy ? sortInsightItems(keywordInsight, sortBy) : keywordInsight; +}; + +export const getPagesInsight = (SCData:SCDomainDataType, sortBy:string = 'clicks', queryDate:string = 'thirtyDays') : SCInsightItem[] => { + const pagesItems: { [key:string]: SCInsightItem } = {}; + const keywordCounts: { [key:string]: number } = {}; + const countriesCount: { [key:string]: string[] } = {}; + const dateKey = queryDate as keyof SCDomainDataType; + const scData = SCData[dateKey] ? SCData[dateKey] as SearchAnalyticsItem[] : []; + const allPages: string[] = [...new Set(scData.map((item) => item.page))]; + + allPages.forEach((page:string) => { + const itemData = { clicks: 0, impressions: 0, ctr: 0, position: 0 }; + scData.forEach((itm) => { + if (itm.page === page) { + itemData.clicks += itm.clicks; + itemData.impressions += itm.impressions; + itemData.ctr += itm.ctr; + itemData.position += itm.position; + if (!countriesCount[page]) { + countriesCount[page] = []; + } + if (countriesCount[page] && !countriesCount[page].includes(itm.country)) { + countriesCount[page].push(itm.country); + } + keywordCounts[page] = keywordCounts[page] ? keywordCounts[page] + 1 : 1; + } + }); + pagesItems[page] = itemData; + }); + + const pagesInsight: SCInsightItem[] = Object.keys(pagesItems).map((page:string) => { + return { + ...pagesItems[page], + position: Math.round(pagesItems[page].position / keywordCounts[page]), + ctr: pagesItems[page].ctr / keywordCounts[page], + countries: countriesCount[page].length, + keywords: keywordCounts[page], + page, + }; + }); + + return sortBy ? sortInsightItems(pagesInsight, sortBy) : pagesInsight; +}; diff --git a/utils/searchConsole.ts b/utils/searchConsole.ts new file mode 100644 index 0000000..98641ce --- /dev/null +++ b/utils/searchConsole.ts @@ -0,0 +1,154 @@ +import { auth, searchconsole_v1 } from '@googleapis/searchconsole'; +import { readFile, writeFile } from 'fs/promises'; +import { getCountryCodeFromAlphaThree } from './countries'; + +export type SCDomainFetchError = { + error: boolean, + errorMsg: string, +} +type fetchConsoleDataResponse = SearchAnalyticsItem[] | SearchAnalyticsStat[] | SCDomainFetchError; +const fetchSearchConsoleData = async (domainName:string, days:number, type?:string): Promise => { + if (!domainName) return { error: true, errorMsg: 'Domain Not Provided!' }; + try { + const authClient = new auth.GoogleAuth({ + credentials: { + private_key: process.env.SEARCH_CONSOLE_PRIVATE_KEY ? process.env.SEARCH_CONSOLE_PRIVATE_KEY.replaceAll('\\n', '\n') : '', + client_email: process.env.SEARCH_CONSOLE_CLIENT_EMAIL ? process.env.SEARCH_CONSOLE_CLIENT_EMAIL : '', + }, + scopes: [ + 'https://www.googleapis.com/auth/webmasters.readonly', + ], + }); + const startDateRaw = new Date(new Date().setDate(new Date().getDate() - days)); + const padDate = (num:number) => String(num).padStart(2, '0'); + const startDate = `${startDateRaw.getFullYear()}-${padDate(startDateRaw.getMonth() + 1)}-${padDate(startDateRaw.getDate())}`; + const endDate = `${new Date().getFullYear()}-${padDate(new Date().getMonth() + 1)}-${padDate(new Date().getDate())}`; + const client = new searchconsole_v1.Searchconsole({ auth: authClient }); + // Params: https://developers.google.com/webmaster-tools/v1/searchanalytics/query + let requestBody:any = { + startDate, + endDate, + type: 'web', + rowLimit: 1000, + dataState: 'all', + dimensions: ['query', 'device', 'country', 'page'], + }; + if (type === 'stat') { + requestBody = { + startDate, + endDate, + dataState: 'all', + dimensions: ['date'], + }; + } + + const res = client.searchanalytics.query({ siteUrl: `sc-domain:${domainName}`, requestBody }); + const resData:any = (await res).data; + let finalRows = resData.rows ? resData.rows.map((item:SearchAnalyticsRawItem) => parseSearchConsoleItem(item, domainName)) : []; + + if (type === 'stat' && resData.rows && resData.rows.length > 0) { + // console.log(resData.rows); + finalRows = []; + resData.rows.forEach((row:SearchAnalyticsRawItem) => { + finalRows.push({ + date: row.keys[0], + clicks: row.clicks, + impressions: row.impressions, + ctr: row.ctr, + position: row.position, + }); + }); + } + + return finalRows; + } catch (error:any) { + const qType = type === 'stats' ? '(stats)' : `(${days}days)`; + console.log(`[ERROR] Search Console API Error for ${domainName} ${qType} : `, error?.response?.status, error?.response?.statusText); + return { error: true, errorMsg: `${error?.response?.status}: ${error?.response?.statusText}` }; + } +}; + +export const fetchDomainSCData = async (domain:string): Promise => { + const days = [3, 7, 30]; + const scDomainData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '', stats: [] }; + if (domain) { + for (const day of days) { + const items = await fetchSearchConsoleData(domain, day); + scDomainData.lastFetched = new Date().toJSON(); + if (Array.isArray(items)) { + if (day === 3) scDomainData.threeDays = items as SearchAnalyticsItem[]; + if (day === 7) scDomainData.sevenDays = items as SearchAnalyticsItem[]; + if (day === 30) scDomainData.thirtyDays = items as SearchAnalyticsItem[]; + } else if (items.error) { + scDomainData.lastFetchError = items.errorMsg; + } + } + const stats = await fetchSearchConsoleData(domain, 30, 'stat'); + if (stats && Array.isArray(stats) && stats.length > 0) { + scDomainData.stats = stats as SearchAnalyticsStat[]; + } + await updateLocalSCData(domain, scDomainData); + } + + return scDomainData; +}; + +export const parseSearchConsoleItem = (SCItem: SearchAnalyticsRawItem, domainName: string): SearchAnalyticsItem => { + const { clicks = 0, impressions = 0, ctr = 0, position = 0 } = SCItem; + const keyword = SCItem.keys[0]; + const device = SCItem.keys[1] ? SCItem.keys[1].toLowerCase() : 'desktop'; + const country = SCItem.keys[2] ? (getCountryCodeFromAlphaThree(SCItem.keys[2].toUpperCase()) || SCItem.keys[2]) : 'ZZ'; + const page = SCItem.keys[3] ? SCItem.keys[3].replace('https://', '').replace('http://', '').replace('www', '').replace(domainName, '') : ''; + const uid = `${country.toLowerCase()}:${device}:${keyword.replaceAll(' ', '_')}`; + + return { keyword, uid, device, country, clicks, impressions, ctr, position, page }; +}; + +export const integrateKeywordSCData = (keyword: KeywordType, SCData:SCDomainDataType) : KeywordType => { + const kuid = `${keyword.country.toLowerCase()}:${keyword.device}:${keyword.keyword.replaceAll(' ', '_')}`; + const impressions:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 }; + const visits :any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 }; + const ctr:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 }; + const position:any = { yesterday: 0, threeDays: 0, sevenDays: 0, thirtyDays: 0, avgSevenDays: 0, avgThreeDays: 0, avgThirtyDays: 0 }; + + const threeDaysData = SCData.threeDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {}; + const SevenDaysData = SCData.sevenDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {}; + const ThirdyDaysData = SCData.thirtyDays.find((item:SearchAnalyticsItem) => item.uid === kuid) || {}; + const totalData:any = { threeDays: threeDaysData, sevenDays: SevenDaysData, thirtyDays: ThirdyDaysData }; + + Object.keys(totalData).forEach((dataKey) => { + let avgDataKey = 'avgThreeDays'; let divideBy = 3; + if (dataKey === 'sevenDays') { avgDataKey = 'avgSevenDays'; divideBy = 7; } + if (dataKey === 'thirtyDays') { avgDataKey = 'avgThirtyDays'; divideBy = 30; } + // Actual Data + impressions[dataKey] = totalData[dataKey].impressions || 0; + visits[dataKey] = totalData[dataKey].clicks || 0; + ctr[dataKey] = Math.round((totalData[dataKey].ctr || 0) * 100) / 100; + position[dataKey] = totalData[dataKey].position ? Math.round(totalData[dataKey].position) : 0; + // Average Data + impressions[avgDataKey] = Math.round(impressions[dataKey] / divideBy); + ctr[avgDataKey] = Math.round((ctr[dataKey] / divideBy) * 100) / 100; + visits[avgDataKey] = Math.round(visits[dataKey] / divideBy); + position[avgDataKey] = Math.round(position[dataKey] / divideBy); + }); + const finalSCData = { impressions, visits, ctr, position }; + + return { ...keyword, scData: finalSCData }; +}; + +export const readLocalSCData = async (domain:string): Promise => { + const filePath = `${process.cwd()}/data/SC_${domain}.json`; + const currentQueueRaw = await readFile(filePath, { encoding: 'utf-8' }).catch(async () => { await updateLocalSCData(domain); return '{}'; }); + const domainSCData = JSON.parse(currentQueueRaw); + return domainSCData; +}; + +export const updateLocalSCData = async (domain:string, scDomainData?:SCDomainDataType): Promise => { + const filePath = `${process.cwd()}/data/SC_${domain}.json`; + const fileData = JSON.stringify(scDomainData); + const emptyData:SCDomainDataType = { threeDays: [], sevenDays: [], thirtyDays: [], lastFetched: '', lastFetchError: '' }; + await writeFile(filePath, fileData, { encoding: 'utf-8' }).catch((err) => { console.log(err); return JSON.stringify(emptyData); }); + return scDomainData || emptyData; +}; + +export default fetchSearchConsoleData; diff --git a/utils/sortFilter.ts b/utils/sortFilter.ts index 187ac02..cf1ee77 100644 --- a/utils/sortFilter.ts +++ b/utils/sortFilter.ts @@ -4,8 +4,8 @@ * @param {string} sortBy - The sort method. * @returns {KeywordType[]} */ -export const sortKeywords = (theKeywords:KeywordType[], sortBy:string) : KeywordType[] => { - let sortedItems = []; +export const sortKeywords = (theKeywords:KeywordType[], sortBy:string, scDataType?: string) : KeywordType[] => { + let sortedItems: KeywordType[] = []; const keywords = theKeywords.map((k) => ({ ...k, position: k.position === 0 ? 111 : k.position })); switch (sortBy) { case 'date_asc': @@ -28,6 +28,42 @@ export const sortKeywords = (theKeywords:KeywordType[], sortBy:string) : Keyword case 'alpha_desc': sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => (a.keyword > b.keyword ? 1 : -1)); break; + case 'imp_asc': + if (scDataType) { + sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => { + const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0; + const aImpressionData = a.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0; + return aImpressionData > bImpressionData ? 1 : -1; + }); + } + break; + case 'imp_desc': + if (scDataType) { + sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => { + const bImpressionData = b.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0; + const aImpressionData = a.scData?.impressions[scDataType as keyof KeywordSCDataChild] || 0; + return bImpressionData > aImpressionData ? 1 : -1; + }); + } + break; + case 'visits_asc': + if (scDataType) { + sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => { + const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0; + const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0; + return aImpressionData > bImpressionData ? 1 : -1; + }); + } + break; + case 'visits_desc': + if (scDataType) { + sortedItems = theKeywords.sort((a: KeywordType, b: KeywordType) => { + const bImpressionData = b.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0; + const aImpressionData = a.scData?.visits[scDataType as keyof KeywordSCDataChild] || 0; + return bImpressionData > aImpressionData ? 1 : -1; + }); + } + break; default: return theKeywords; } diff --git a/utils/verifyUser.ts b/utils/verifyUser.ts index 4eb5629..eb4dec2 100644 --- a/utils/verifyUser.ts +++ b/utils/verifyUser.ts @@ -13,7 +13,17 @@ const verifyUser = (req: NextApiRequest, res: NextApiResponse): string => { const cookies = new Cookies(req, res); const token = cookies && cookies.get('token'); - const allowedApiRoutes = ['GET:/api/keyword', 'GET:/api/keywords', 'GET:/api/domains', 'POST:/api/refresh', 'POST:/api/cron', 'POST:/api/notify']; + const allowedApiRoutes = [ + 'GET:/api/keyword', + 'GET:/api/keywords', + 'GET:/api/domains', + 'POST:/api/refresh', + 'POST:/api/cron', + 'POST:/api/notify', + 'POST:/api/searchconsole', + 'GET:/api/searchconsole', + 'GET:/api/insight', + ]; const verifiedAPI = req.headers.authorization ? req.headers.authorization.substring('Bearer '.length) === process.env.APIKEY : false; const accessingAllowedRoute = req.url && req.method && allowedApiRoutes.includes(`${req.method}:${req.url.replace(/\?(.*)/, '')}`); console.log(req.method, req.url);