-
-
Notifications
You must be signed in to change notification settings - Fork 1.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
feat: Refactor notifications and make them injectable #2034
Changes from all commits
33ead86
a73a18a
dde6c7e
ebd0886
534f83c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,23 @@ | ||
import {Link, Store} from '../store/model'; | ||
|
||
type RequestFailed = { | ||
failureReason: 'request_failed'; | ||
statusCode: number; | ||
}; | ||
type Captcha = {failureReason: 'captcha'}; | ||
type MaxPriceExceeded = {failureReason: 'max_price'; maxPrice: number}; | ||
type OutOfStock = {failureReason: 'out_of_stock'}; | ||
type BannedSeller = {failureReason: 'banned_seller'}; | ||
type LinkPollError = {result: 'failure'} & ( | ||
| RequestFailed | ||
| Captcha | ||
| MaxPriceExceeded | ||
| OutOfStock | ||
| BannedSeller | ||
); | ||
type LinkPollSuccess = {result: 'in_stock'; url: string}; | ||
|
||
export type LinkPollEvent = (LinkPollError | LinkPollSuccess) & { | ||
link: Link; | ||
store: Store; | ||
}; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The I wasn't sure if events should be provided for |
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
import {Print, logger} from '../logger'; | ||
import {LinkPollEvent} from './link_poll_event'; | ||
|
||
function assertNever(x: never): never { | ||
throw new Error('Unexpected object: ' + x); | ||
} | ||
|
||
export function sendLogstream(pollEvent: LinkPollEvent) { | ||
const {link, store} = pollEvent; | ||
if (pollEvent.result === 'in_stock') { | ||
logger.info(`${Print.inStock(link, store, true)}\n${pollEvent.url}`); | ||
} else { | ||
switch (pollEvent.failureReason) { | ||
case 'captcha': | ||
logger.warn(Print.captcha(link, store, true)); | ||
return; | ||
case 'banned_seller': | ||
logger.warn(Print.bannedSeller(link, store, true)); | ||
return; | ||
case 'out_of_stock': | ||
logger.info(Print.outOfStock(link, store, true)); | ||
return; | ||
case 'max_price': | ||
logger.info(Print.maxPrice(link, store, pollEvent.maxPrice, true)); | ||
return; | ||
case 'request_failed': | ||
return; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I think this case logs too but I haven't found where it happens. |
||
default: | ||
assertNever(pollEvent); | ||
} | ||
} | ||
} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. All the logging w.r.t polling had been moved here. Some logging still happends in |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -22,13 +22,15 @@ import {fetchLinks} from './fetch-links'; | |
import {filterStoreLink} from './filter'; | ||
import open from 'open'; | ||
import {processBackoffDelay} from './model/helpers/backoff'; | ||
import {sendNotification} from '../notification'; | ||
import useProxy from '@doridian/puppeteer-page-proxy'; | ||
import {LinkPollEvent} from '../notification/link_poll_event'; | ||
|
||
const inStock: Record<string, boolean> = {}; | ||
|
||
const linkBuilderLastRunTimes: Record<string, number> = {}; | ||
|
||
export type SendNotification = (pollEvent: LinkPollEvent) => void; | ||
|
||
function nextProxy(store: Store) { | ||
if (!store.proxyList) { | ||
return; | ||
|
@@ -151,7 +153,11 @@ async function handleAdBlock(request: HTTPRequest, adBlockRequestHandler: any) { | |
* @param browser Puppeteer browser. | ||
* @param store Vendor of graphics cards. | ||
*/ | ||
async function lookup(browser: Browser, store: Store) { | ||
async function lookup( | ||
browser: Browser, | ||
store: Store, | ||
sendNotification: SendNotification | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Most of the added code in this file is passing the The rest is sending notifications in lieu of logging |
||
) { | ||
if (!getStores().has(store.name)) { | ||
return; | ||
} | ||
|
@@ -252,7 +258,13 @@ async function lookup(browser: Browser, store: Store) { | |
let statusCode = 0; | ||
|
||
try { | ||
statusCode = await lookupCard(browser, store, page, link); | ||
statusCode = await lookupCard( | ||
browser, | ||
store, | ||
page, | ||
link, | ||
sendNotification | ||
); | ||
} catch (error: unknown) { | ||
if (store.currentProxyIndex !== undefined && store.proxyList) { | ||
const proxy = `${store.currentProxyIndex + 1}/${ | ||
|
@@ -294,7 +306,8 @@ async function lookupCard( | |
browser: Browser, | ||
store: Store, | ||
page: Page, | ||
link: Link | ||
link: Link, | ||
sendNotification: SendNotification | ||
): Promise<number> { | ||
const givenWaitFor = store.waitUntil ? store.waitUntil : 'networkidle0'; | ||
const response: HTTPResponse | null = await page.goto(link.url, { | ||
|
@@ -305,21 +318,32 @@ async function lookupCard( | |
const statusCode = await handleResponse(browser, store, page, link, response); | ||
|
||
if (!isStatusCodeInRange(statusCode, successStatusCodes)) { | ||
sendNotification({ | ||
result: 'failure', | ||
failureReason: 'request_failed', | ||
statusCode, | ||
link, | ||
store, | ||
}); | ||
return statusCode; | ||
} | ||
|
||
if (await lookupCardInStock(store, page, link)) { | ||
if (await lookupCardInStock(store, page, link, sendNotification)) { | ||
const givenUrl = | ||
link.cartUrl && config.store.autoAddToCart ? link.cartUrl : link.url; | ||
logger.info(`${Print.inStock(link, store, true)}\n${givenUrl}`); | ||
|
||
if (config.browser.open) { | ||
await (link.openCartAction === undefined | ||
? open(givenUrl) | ||
: link.openCartAction(browser)); | ||
} | ||
|
||
sendNotification(link, store); | ||
sendNotification({ | ||
result: 'in_stock', | ||
url: givenUrl, | ||
link, | ||
store, | ||
}); | ||
|
||
if (config.page.inStockWaitTime) { | ||
inStock[link.url] = true; | ||
|
@@ -407,7 +431,12 @@ async function checkIsCloudflare(store: Store, page: Page, link: Link) { | |
return false; | ||
} | ||
|
||
async function lookupCardInStock(store: Store, page: Page, link: Link) { | ||
async function lookupCardInStock( | ||
store: Store, | ||
page: Page, | ||
link: Link, | ||
sendNotification: SendNotification | ||
) { | ||
const baseOptions: Selector = { | ||
requireVisible: false, | ||
selector: store.labels.container ?? 'body', | ||
|
@@ -416,7 +445,12 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { | |
|
||
if (store.labels.captcha) { | ||
if (await pageIncludesLabels(page, store.labels.captcha, baseOptions)) { | ||
logger.warn(Print.captcha(link, store, true)); | ||
sendNotification({ | ||
result: 'failure', | ||
failureReason: 'captcha', | ||
link, | ||
store, | ||
}); | ||
await delay(getSleepTime(store)); | ||
return false; | ||
} | ||
|
@@ -426,14 +460,24 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { | |
if ( | ||
await pageIncludesLabels(page, store.labels.bannedSeller, baseOptions) | ||
) { | ||
logger.warn(Print.bannedSeller(link, store, true)); | ||
sendNotification({ | ||
result: 'failure', | ||
failureReason: 'banned_seller', | ||
link, | ||
store, | ||
}); | ||
return false; | ||
} | ||
} | ||
|
||
if (store.labels.outOfStock) { | ||
if (await pageIncludesLabels(page, store.labels.outOfStock, baseOptions)) { | ||
logger.info(Print.outOfStock(link, store, true)); | ||
sendNotification({ | ||
result: 'failure', | ||
failureReason: 'out_of_stock', | ||
link, | ||
store, | ||
}); | ||
return false; | ||
} | ||
} | ||
|
@@ -444,7 +488,13 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { | |
link.price = await getPrice(page, store.labels.maxPrice, baseOptions); | ||
|
||
if (link.price && link.price > maxPrice && maxPrice > 0) { | ||
logger.info(Print.maxPrice(link, store, maxPrice, true)); | ||
sendNotification({ | ||
result: 'failure', | ||
failureReason: 'max_price', | ||
maxPrice, | ||
link, | ||
store, | ||
}); | ||
return false; | ||
} | ||
} | ||
|
@@ -466,7 +516,12 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { | |
}; | ||
|
||
if (!(await pageIncludesLabels(page, store.labels.inStock, options))) { | ||
logger.info(Print.outOfStock(link, store, true)); | ||
sendNotification({ | ||
result: 'failure', | ||
failureReason: 'out_of_stock', | ||
link, | ||
store, | ||
}); | ||
return false; | ||
} | ||
} | ||
|
@@ -479,7 +534,12 @@ async function lookupCardInStock(store: Store, page: Page, link: Link) { | |
}; | ||
|
||
if (!(await pageIncludesLabels(page, link.labels.inStock, options))) { | ||
logger.info(Print.outOfStock(link, store, true)); | ||
sendNotification({ | ||
result: 'failure', | ||
failureReason: 'out_of_stock', | ||
link, | ||
store, | ||
}); | ||
return false; | ||
} | ||
} | ||
|
@@ -540,20 +600,24 @@ async function runCaptchaDeterrent(browser: Browser, store: Store, page: Page) { | |
} | ||
} | ||
|
||
export async function tryLookupAndLoop(browser: Browser, store: Store) { | ||
export async function tryLookupAndLoop( | ||
browser: Browser, | ||
store: Store, | ||
sendNotification: SendNotification | ||
) { | ||
if (!browser.isConnected()) { | ||
logger.debug(`[${store.name}] Ending this loop as browser is disposed...`); | ||
return; | ||
} | ||
|
||
logger.debug(`[${store.name}] Starting lookup...`); | ||
try { | ||
await lookup(browser, store); | ||
await lookup(browser, store, sendNotification); | ||
} catch (error: unknown) { | ||
logger.error(error); | ||
} | ||
|
||
const sleepTime = getSleepTime(store); | ||
logger.debug(`[${store.name}] Lookup done, next one in ${sleepTime} ms`); | ||
setTimeout(tryLookupAndLoop, sleepTime, browser, store); | ||
setTimeout(tryLookupAndLoop, sleepTime, browser, store, sendNotification); | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
In the
main
function we inject the existingsendNotification
function