Skip to content

Commit

Permalink
some refactoring in flow of url/script/post-process-script
Browse files Browse the repository at this point in the history
  • Loading branch information
@martinrotter
martinrotter committed Oct 16, 2023
1 parent 8720fe6 commit 6100e56
Show file tree
Hide file tree
Showing 3 changed files with 36 additions and 36 deletions.
22 changes: 11 additions & 11 deletions src/librssguard/services/standard/standardfeed.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -268,13 +268,13 @@ StandardFeed* StandardFeed::guessFeed(StandardFeed::SourceType source_type,
qDebugNN << LOGSEC_CORE << "Running custom script for guessing" << QUOTE_W_SPACE(source) << "to obtain feed data.";

// Use script to generate feed file.
feed_contents = generateFeedFileWithScript(source, timeout).toUtf8();
feed_contents = generateFeedFileWithScript(source, timeout);
}

if (!post_process_script.simplified().isEmpty()) {
qDebugNN << LOGSEC_CORE << "Post-processing obtained feed data with custom script for guessing"
<< QUOTE_W_SPACE_DOT(post_process_script);
feed_contents = postProcessFeedFileWithScript(post_process_script, feed_contents, timeout).toUtf8();
feed_contents = postProcessFeedFileWithScript(post_process_script, feed_contents, timeout);
}

StandardFeed* feed = nullptr;
Expand Down Expand Up @@ -382,11 +382,11 @@ QStringList StandardFeed::prepareExecutionLine(const QString& execution_line) {
return qApp->replaceDataUserDataFolderPlaceholder(args);
}

QString StandardFeed::runScriptProcess(const QStringList& cmd_args,
const QString& working_directory,
int run_timeout,
bool provide_input,
const QString& input) {
QByteArray StandardFeed::runScriptProcess(const QStringList& cmd_args,
const QString& working_directory,
int run_timeout,
bool provide_input,
const QString& input) {
QProcess process;

if (provide_input) {
Expand Down Expand Up @@ -450,7 +450,7 @@ QString StandardFeed::runScriptProcess(const QStringList& cmd_args,
}
}

QString StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) {
QByteArray StandardFeed::generateFeedFileWithScript(const QString& execution_line, int run_timeout) {
auto prepared_query = prepareExecutionLine(execution_line);

if (prepared_query.isEmpty()) {
Expand All @@ -460,9 +460,9 @@ QString StandardFeed::generateFeedFileWithScript(const QString& execution_line,
return runScriptProcess(prepared_query, qApp->userDataFolder(), run_timeout, false);
}

QString StandardFeed::postProcessFeedFileWithScript(const QString& execution_line,
const QString& raw_feed_data,
int run_timeout) {
QByteArray StandardFeed::postProcessFeedFileWithScript(const QString& execution_line,
const QString& raw_feed_data,
int run_timeout) {
auto prepared_query = prepareExecutionLine(execution_line);

if (prepared_query.isEmpty()) {
Expand Down
18 changes: 9 additions & 9 deletions src/librssguard/services/standard/standardfeed.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -94,15 +94,15 @@ class StandardFeed : public Feed {

// Scraping + post+processing.
static QStringList prepareExecutionLine(const QString& execution_line);
static QString generateFeedFileWithScript(const QString& execution_line, int run_timeout);
static QString postProcessFeedFileWithScript(const QString& execution_line,
const QString& raw_feed_data,
int run_timeout);
static QString runScriptProcess(const QStringList& cmd_args,
const QString& working_directory,
int run_timeout,
bool provide_input,
const QString& input = {});
static QByteArray generateFeedFileWithScript(const QString& execution_line, int run_timeout);
static QByteArray postProcessFeedFileWithScript(const QString& execution_line,
const QString& raw_feed_data,
int run_timeout);
static QByteArray runScriptProcess(const QStringList& cmd_args,
const QString& working_directory,
int run_timeout,
bool provide_input,
const QString& input = {});

public slots:
void fetchMetadataForItself();
Expand Down
32 changes: 16 additions & 16 deletions src/librssguard/services/standard/standardserviceroot.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -156,13 +156,13 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
Q_UNUSED(tagged_messages)

StandardFeed* f = static_cast<StandardFeed*>(feed);
QByteArray feed_contents;
QString formatted_feed_contents;
int download_timeout = qApp->settings()->value(GROUP(Feeds), SETTING(Feeds::UpdateTimeout)).toInt();

if (f->sourceType() == StandardFeed::SourceType::Url) {
qDebugNN << LOGSEC_CORE << "Downloading URL" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";

QByteArray feed_contents;
QList<QPair<QByteArray, QByteArray>> headers;

headers << NetworkFactory::generateBasicAuthHeader(f->protection(), f->username(), f->password());
Expand Down Expand Up @@ -198,25 +198,13 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
qWarningNN << LOGSEC_CORE << "This feed is gzipped.";
#endif
}

// Encode downloaded data for further parsing.
QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit());

if (codec == nullptr) {
// No suitable codec for this encoding was found.
// Use non-converted data.
formatted_feed_contents = feed_contents;
}
else {
formatted_feed_contents = codec->toUnicode(feed_contents);
}
}
else {
qDebugNN << LOGSEC_CORE << "Running custom script" << QUOTE_W_SPACE(feed->source()) << "to obtain feed data.";

// Use script to generate feed file.
try {
formatted_feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout);
feed_contents = StandardFeed::generateFeedFileWithScript(feed->source(), download_timeout);
}
catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE << "Custom script for generating feed file failed:" << QUOTE_W_SPACE_DOT(ex.message());
Expand All @@ -230,8 +218,8 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
<< QUOTE_W_SPACE_DOT(f->postProcessScript());

try {
formatted_feed_contents =
StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(), formatted_feed_contents, download_timeout);
feed_contents =
StandardFeed::postProcessFeedFileWithScript(f->postProcessScript(), feed_contents, download_timeout);
}
catch (const ScriptException& ex) {
qCriticalNN << LOGSEC_CORE << "Post-processing script for feed file failed:" << QUOTE_W_SPACE_DOT(ex.message());
Expand All @@ -240,6 +228,18 @@ QList<Message> StandardServiceRoot::obtainNewMessages(Feed* feed,
}
}

// Encode obtained data for further parsing.
QTextCodec* codec = QTextCodec::codecForName(f->encoding().toLocal8Bit());

if (codec == nullptr) {
// No suitable codec for this encoding was found.
// Use UTF-8.
formatted_feed_contents = QString::fromUtf8(feed_contents);
}
else {
formatted_feed_contents = codec->toUnicode(feed_contents);
}

// Feed data are downloaded and encoded.
// Parse data and obtain messages.
QList<Message> messages;
Expand Down

0 comments on commit 6100e56

Please sign in to comment.