Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
thekavikumar committed Jan 26, 2024
1 parent 43c3751 commit 96d5256
Showing 1 changed file with 14 additions and 4 deletions.
18 changes: 14 additions & 4 deletions lib/training/wiki_training_loader.rb
Original file line number Diff line number Diff line change
Expand Up @@ -96,12 +96,22 @@ def wiki_source_pages
# To handle more than 500 pages linked from the source page,
# we'll need to update this to use 'continue'.
query_params = { prop: 'links', titles: @wiki_base_page, pllimit: 500 }
response = WikiApi.new(MetaWiki.new).query(query_params)
links = []
begin
response.data['pages'].values[0]['links'].map { |page| page['title'] }
rescue StandardError
raise InvalidWikiContentError, "could not get links from '#{@wiki_base_page}'"
response = WikiApi.new(MetaWiki.new).query(query_params)
loop do
current_links = response.dig('pages', @wiki_base_page, 'links') || []
links.concat(current_links.map { |page| page['title'] })

@continue = response['continue']&.fetch('plcontinue', 'done')
break if @continue == 'done'

response = WikiApi.new(MetaWiki.new).query(query_params.merge(plcontinue: @continue))
end
rescue StandardError => e
raise InvalidWikiContentError, "could not get links from '#{@wiki_base_page}': #{e.message}"
end
links
end

def listed_wiki_source_pages
Expand Down

0 comments on commit 96d5256

Please sign in to comment.