Skip to content

Commit

Permalink
Merge pull request #122 from ontoportal-lirmm/pr/fix/append_triples_b…
Browse files Browse the repository at this point in the history
…y_chunks

Feature: Append new submission data by chunks to the triple store
alexskr authored Mar 14, 2024
2 parents f00c8f8 + 225c144 commit 5caeb0d
Showing 1 changed file with 43 additions and 27 deletions.
70 changes: 43 additions & 27 deletions lib/goo/sparql/client.rb
Original file line number Diff line number Diff line change
@@ -83,41 +83,28 @@ def delete_data_graph(graph)
def append_triples_no_bnodes(graph,file_path,mime_type_in)
bnodes_filter = nil
dir = nil

if file_path.end_with?("ttl")
response = nil
if file_path.end_with?('ttl')
bnodes_filter = file_path
else
bnodes_filter,dir = bnodes_filter_file(file_path,mime_type_in)
end
mime_type = "text/turtle"

if mime_type_in == "text/x-nquads"
mime_type = "text/x-nquads"
graph = "http://data.bogus.graph/uri"
bnodes_filter, dir = bnodes_filter_file(file_path, mime_type_in)
end
chunk_lines = 500_000 # number of line
file = File.foreach(bnodes_filter)
lines = []
file.each_entry do |line|
lines << line
if lines.size == chunk_lines
response = execute_append_request graph, lines.join, mime_type_in
lines.clear
end
end

data_file = File.read(bnodes_filter)
params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil}
backend_name = Goo.sparql_backend_name

if backend_name == BACKEND_4STORE
params[:payload] = {
graph: graph.to_s,
data: data_file,
"mime-type" => mime_type
}
#for some reason \\\\ breaks parsing
params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n")
else
params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}"
params[:payload] = data_file
end
response = execute_append_request graph, lines.join, mime_type_in unless lines.empty?

response = RestClient::Request.execute(params)

unless dir.nil?
File.delete(bnodes_filter)

begin
FileUtils.rm_rf(dir)
rescue => e
@@ -196,6 +183,35 @@ def status
resp[:outstanding] = outstanding
resp
end

private

def execute_append_request(graph, data_file, mime_type_in)
mime_type = "text/turtle"

if mime_type_in == "text/x-nquads"
mime_type = "text/x-nquads"
graph = "http://data.bogus.graph/uri"
end

params = {method: :post, url: "#{url.to_s}", headers: {"content-type" => mime_type, "mime-type" => mime_type}, timeout: nil}
backend_name = Goo.sparql_backend_name

if backend_name == BACKEND_4STORE
params[:payload] = {
graph: graph.to_s,
data: data_file,
'mime-type' => mime_type
}
#for some reason \\\\ breaks parsing
params[:payload][:data] = params[:payload][:data].split("\n").map { |x| x.sub("\\\\","") }.join("\n")
else
params[:url] << "?context=#{CGI.escape("<#{graph.to_s}>")}"
params[:payload] = data_file
end

RestClient::Request.execute(params)
end
end
end
end

0 comments on commit 5caeb0d

Please sign in to comment.