Support for POST, better logging and error handling, sparql-results+j…

…son now default, command-line argument --access-token (#657)

1. There is now proper support also for POST queries. The POST queries allow both variants specified in the SPARQL 1.1 standard: with a URL-encoded query string ("query=...") and with content type "application/sparql-query" and just the unencoded SPARQL query in the body of the POST request.

2. Make the logging and handling of errors more consistent. The standard should be that the error message in the log is equivalent to that send to the server. Use the occasion to improve the server logging in general. For example, the server now shows the method of the query, the requested media type, the time needed for query planning, and the total query processing time. Also, the query planning now comes after determining the media type, as it should.

3. Make sparql-results+json the default media type, as required by the standard. This is possible now because the latest version of the QLever UI explicitly asks for qlever-results+json for every SPARQL query it sends.

4. Add a command-line argument for specifying the access token to be used for restricted API calls like clear-cache complete. This was simple enough and completes work from an earlier PR. Also support an explicit "ping" command.

e2e/e2e.sh

@@ -112,8 +112,8 @@ if [ ${REBUILD_THE_INDEX} == "YES" ] || ! [ -f "${INDEX}.vocabulary" ]; then
 	popd
 fi
 
-# Launch the Server using the freshly baked index. Can't simply use a subshell here because
-# then we can't easily get the SERVER_PID out of that subshell
+# Launch the Server using the freshly baked index. Can't simply use a subshell
+# here because then we can't easily get the SERVER_PID out of that subshell
 pushd "$BINARY_DIR"
 echo "Launching server from path $(pwd)"
 ./ServerMain -i "$INDEX" -p 9099 -m 1 -t &> server_log.txt &

src/ServerMain.cpp

@@ -39,7 +39,8 @@ int main(int argc, char** argv) {
   // filled / set depending on the options.
   using ad_utility::NonNegative;
 
-  string indexBasename;
+  std::string indexBasename;
+  std::string accessToken;
   bool text = false;
   int port;
   NonNegative numSimultaneousQueries = 1;
@@ -62,6 +63,8 @@ int main(int argc, char** argv) {
       "The basename of the index files (required).");
   add("port,p", po::value<int>(&port)->required(),
       "The port on which HTTP requests are served (required).");
+  add("access-token,a", po::value<std::string>(&accessToken)->default_value(""),
+      "Access token for restricted API calls (default: no access).");
   add("num-simultaneous-queries,j",
       po::value<NonNegative>(&numSimultaneousQueries)->default_value(1),
       "The number of queries that can be processed simultaneously.");
@@ -123,7 +126,7 @@ int main(int argc, char** argv) {
 
   try {
     Server server(port, static_cast<int>(numSimultaneousQueries),
-                  memoryMaxSizeGb);
+                  memoryMaxSizeGb, std::move(accessToken));
     server.run(indexBasename, text, !noPatterns, !noPatternTrick,
                !onlyPsoAndPosPermutations);
   } catch (const std::exception& e) {

src/engine/Operation.cpp

@@ -56,9 +56,8 @@ void Operation::recursivelySetTimeoutTimer(
   }
 }
 
-// Get the result for the subtree rooted at this element.
-// Use existing results if they are already available, otherwise
-// trigger computation.
+// Get the result for the subtree rooted at this element. Use existing results
+// if they are already available, otherwise trigger computation.
 shared_ptr<const ResultTable> Operation::getResult(bool isRoot) {
   ad_utility::Timer timer;
   timer.start();
@@ -114,30 +113,34 @@ shared_ptr<const ResultTable> Operation::getResult(bool isRoot) {
 
     timer.stop();
     createRuntimeInformation(result, timer.msecs());
+    auto resultNumRows = result._resultPointer->_resultTable->size();
+    auto resultNumCols = result._resultPointer->_resultTable->width();
+    LOG(DEBUG) << "Computed result of size " << resultNumRows << " x "
+               << resultNumCols << std::endl;
     return result._resultPointer->_resultTable;
   } catch (const ad_semsearch::AbortException& e) {
     // A child Operation was aborted, do not print the information again.
     throw;
   } catch (const ad_utility::WaitedForResultWhichThenFailedException& e) {
+    // Here and in the following, show the detailed information (it's the
+    // runtime info) only in the DEBUG log. Note that the exception will be
+    // caught by the `processQuery` method, where the error message will be
+    // printed *and* included in an error response sent to the client.
     LOG(ERROR) << "Waited for a result from another thread which then failed"
                << endl;
-    LOG(ERROR) << asString();
+    LOG(DEBUG) << asString();
     throw ad_semsearch::AbortException(e);
   } catch (const std::exception& e) {
     // We are in the innermost level of the exception, so print
-    LOG(ERROR) << "Aborted Operation:" << endl;
-    LOG(ERROR) << asString() << endl;
-    LOG(ERROR) << e.what() << endl;
+    LOG(ERROR) << "Aborted Operation" << endl;
+    LOG(DEBUG) << asString() << endl;
     // Rethrow as QUERY_ABORTED allowing us to print the Operation
     // only at innermost failure of a recursive call
     throw ad_semsearch::AbortException(e);
   } catch (...) {
     // We are in the innermost level of the exception, so print
-    LOG(ERROR) << "Aborted Operation:" << endl;
-    LOG(ERROR) << asString() << endl;
-    LOG(ERROR)
-        << "Unexpected exception that is not a subclass of std::exception"
-        << endl;
+    LOG(ERROR) << "Aborted Operation" << endl;
+    LOG(DEBUG) << asString() << endl;
     // Rethrow as QUERY_ABORTED allowing us to print the Operation
     // only at innermost failure of a recursive call
     throw ad_semsearch::AbortException(

src/engine/QueryExecutionTree.cpp

@@ -401,7 +401,9 @@ ad_utility::streams::stream_generator QueryExecutionTree::generateResults(
   // This call triggers the possibly expensive computation of the query result
   // unless the result is already cached.
   shared_ptr<const ResultTable> resultTable = getResult();
-  LOG(DEBUG) << "Resolving strings for finished binary result...\n";
+  resultTable->logResultSize();
+  LOG(DEBUG) << "Converting result IDs to their corresponding strings ..."
+             << std::endl;
   auto selectedColumnIndices = selectedVariablesToColumnIndices(
       selectedVarsOrAsterisk, *resultTable, true);
 
@@ -518,8 +520,10 @@ QueryExecutionTree::generateRdfGraph(
 // _____________________________________________________________________________
 ad_utility::streams::stream_generator QueryExecutionTree::writeRdfGraphTurtle(
     const ad_utility::sparql_types::Triples& constructTriples, size_t limit,
-    size_t offset, std::shared_ptr<const ResultTable> res) const {
-  auto generator = generateRdfGraph(constructTriples, limit, offset, res);
+    size_t offset, std::shared_ptr<const ResultTable> resultTable) const {
+  resultTable->logResultSize();
+  auto generator =
+      generateRdfGraph(constructTriples, limit, offset, resultTable);
   for (const auto& triple : generator) {
     co_yield triple._subject;
     co_yield ' ';
@@ -535,19 +539,21 @@ template <QueryExecutionTree::ExportSubFormat format>
 ad_utility::streams::stream_generator
 QueryExecutionTree::writeRdfGraphSeparatedValues(
     const ad_utility::sparql_types::Triples& constructTriples, size_t limit,
-    size_t offset, std::shared_ptr<const ResultTable> res) const {
+    size_t offset, std::shared_ptr<const ResultTable> resultTable) const {
   static_assert(format == ExportSubFormat::BINARY ||
                 format == ExportSubFormat::CSV ||
                 format == ExportSubFormat::TSV);
   if constexpr (format == ExportSubFormat::BINARY) {
     throw std::runtime_error{
         "Binary export is not supported for CONSTRUCT queries"};
   }
+  resultTable->logResultSize();
   constexpr auto& escapeFunction = format == ExportSubFormat::TSV
                                        ? RdfEscaping::escapeForTsv
                                        : RdfEscaping::escapeForCsv;
   constexpr char sep = format == ExportSubFormat::TSV ? '\t' : ',';
-  auto generator = generateRdfGraph(constructTriples, limit, offset, res);
+  auto generator =
+      generateRdfGraph(constructTriples, limit, offset, resultTable);
   for (auto& triple : generator) {
     co_yield escapeFunction(std::move(triple._subject));
     co_yield sep;

src/engine/QueryExecutionTree.h

@@ -23,9 +23,8 @@
 using std::shared_ptr;
 using std::string;
 
-// A query execution tree.
-// Processed bottom up, this gives an ordering to the operations
-// needed to solve a query.
+// A query execution tree. Processed bottom up, which gives an ordering to the
+// operations needed to solve a query.
 class QueryExecutionTree {
  public:
   explicit QueryExecutionTree(QueryExecutionContext* const qec);

src/engine/ResultTable.h

@@ -74,6 +74,15 @@ class ResultTable {
   size_t size() const;
   size_t width() const { return _idTable.cols(); }
 
+  // Log to INFO the size of this result.
+  //
+  // NOTE: Due to the current sub-optimal design of `Server::processQuery`, we
+  // need the same message in multiple places and so instead of duplicating the
+  // message, we should have a method for it.
+  void logResultSize() const {
+    LOG(INFO) << "Result has size " << size() << " x " << width() << std::endl;
+  }
+
   void clear();
 
   string asDebugString() const;
@@ -86,4 +95,4 @@ class ResultTable {
   }
 
  private:
-};
+};