defaults.js

const defaultRequestUserAgent = 'website-scraper (https://github.com/website-scraper/node-website-scraper)';

const config = {
	defaultFilename: 'index.html',
	prettifyUrls: false,
	sources: [
		{ selector: 'style' },
		{ selector: '[style]', attr: 'style' },
		{ selector: 'img', attr: 'src' },
		{ selector: 'img', attr: 'srcset' },
		{ selector: 'input', attr: 'src' },
		{ selector: 'object', attr: 'data' },
		{ selector: 'embed', attr: 'src' },
		{ selector: 'param[name="movie"]', attr: 'value' },
		{ selector: 'script', attr: 'src' },
		{ selector: 'link[rel="stylesheet"]', attr: 'href' },
		{ selector: 'link[rel*="icon"]', attr: 'href' },
		{ selector: 'svg *[xlink\\:href]', attr: 'xlink:href' },
		{ selector: 'svg *[href]', attr: 'href' },
		{ selector: 'picture source', attr: 'srcset' },
		{ selector: 'meta[property="og\\:image"]', attr: 'content' },
		{ selector: 'meta[property="og\\:image\\:url"]', attr: 'content' },
		{ selector: 'meta[property="og\\:image\\:secure_url"]', attr: 'content' },
		{ selector: 'meta[property="og\\:audio"]', attr: 'content' },
		{ selector: 'meta[property="og\\:audio\\:url"]', attr: 'content' },
		{ selector: 'meta[property="og\\:audio\\:secure_url"]', attr: 'content' },
		{ selector: 'meta[property="og\\:video"]', attr: 'content' },
		{ selector: 'meta[property="og\\:video\\:url"]', attr: 'content' },
		{ selector: 'meta[property="og\\:video\\:secure_url"]', attr: 'content' },
		{ selector: 'video', attr: 'src' },
		{ selector: 'video source', attr: 'src' },
		{ selector: 'video track', attr: 'src' },
		{ selector: 'audio', attr: 'src' },
		{ selector: 'audio source', attr: 'src' },
		{ selector: 'audio track', attr: 'src' },
		{ selector: 'frame', attr: 'src' },
		{ selector: 'iframe', attr: 'src' },
		{ selector: '[background]', attr: 'background' }
	],
	subdirectories: [
		{ directory: 'images', extensions: ['.png', '.jpg', '.jpeg', '.gif', '.webp'] },
		{ directory: 'js', extensions: ['.js'] },
		{ directory: 'css', extensions: ['.css'] },
		{ directory: 'media', extensions: ['.mp4', '.mp3', '.ogg', '.webm', '.mov', '.wave', '.wav', '.flac'] },
		{ directory: 'fonts', extensions: ['.ttf', '.woff', '.woff2', '.eot', '.svg'] }
	],
	request: {
		throwHttpErrors: false,
		responseType: 'buffer',
		//cookieJar: true,
		decompress: true,
		headers: {
			'user-agent': defaultRequestUserAgent
		},
		https: {
			rejectUnauthorized: false
		}
	},
	requestConcurrency: Infinity,
	urlFilter: null,
	recursive: false,
	maxRecursiveDepth: null,
	maxDepth: null,
	ignoreErrors: false
};

export default config;