Use ElasticSearch 8.5.3 in Github Actions, remove Ubuntu 18

.github/workflows/build-and-test.ubuntu-debug.yml

@@ -8,12 +8,24 @@ jobs:
     name: Build and test in Debug mode
     strategy:
       matrix:
-        os: [ ubuntu-18.04, ubuntu-20.04 ]
+        os: [ ubuntu-20.04 ]
     runs-on: ${{ matrix.os }}
     services:
-      elasticsearch:
-        image: docker://elasticsearch:7.17.7
-        options: --env discovery.type=single-node --publish 9200:9200 --publish 9300:9300
+      elasticsearch8:
+        image: elastic/elasticsearch:8.5.3
+        options: >-
+          --env discovery.type=single-node
+          --env xpack.security.enabled=false
+          --env xpack.security.http.ssl.enabled=false
+          --env action.destructive_requires_name=false
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9200:9200
+      elasticsearch7:
+        image: elastic/elasticsearch:7.17.8
+        options: >-
+          --env discovery.type=single-node
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9201:9200
     steps:
     - name: Install dependencies
       run: |
@@ -105,9 +117,8 @@ jobs:
         _build/tests/app_test -l test_suite
         df -h
         rm -rf /tmp/graphene*
-        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
-        echo
+        _build/tests/es_test -l test_suite
+        export GRAPHENE_TESTING_ES_URL=http://127.0.0.1:9201/
         _build/tests/es_test -l test_suite
         df -h
         rm -rf /tmp/graphene*

.github/workflows/build-and-test.ubuntu-release.yml

@@ -8,12 +8,24 @@ jobs:
     name: Build and test in Release mode
     strategy:
       matrix:
-        os: [ ubuntu-18.04, ubuntu-20.04 ]
+        os: [ ubuntu-20.04 ]
     runs-on: ${{ matrix.os }}
     services:
-      elasticsearch:
-        image: docker://elasticsearch:7.17.7
-        options: --env discovery.type=single-node --publish 9200:9200 --publish 9300:9300
+      elasticsearch8:
+        image: elastic/elasticsearch:8.5.3
+        options: >-
+          --env discovery.type=single-node
+          --env xpack.security.enabled=false
+          --env xpack.security.http.ssl.enabled=false
+          --env action.destructive_requires_name=false
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9200:9200
+      elasticsearch7:
+        image: elastic/elasticsearch:7.17.8
+        options: >-
+          --env discovery.type=single-node
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9201:9200
     steps:
     - name: Install dependencies
       run: |
@@ -73,9 +85,8 @@ jobs:
     - name: Unit-Tests
       run: |
         _build/tests/app_test -l test_suite
-        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
-        echo
+        _build/tests/es_test -l test_suite
+        export GRAPHENE_TESTING_ES_URL=http://127.0.0.1:9201/
         _build/tests/es_test -l test_suite
         libraries/fc/tests/run-parallel-tests.sh _build/tests/chain_test -l test_suite
         _build/tests/cli_test -l test_suite

.github/workflows/sonar-scan.yml

@@ -11,9 +11,15 @@ jobs:
         os: [ ubuntu-20.04 ]
     runs-on: ${{ matrix.os }}
     services:
-      elasticsearch:
-        image: docker://elasticsearch:7.17.7
-        options: --env discovery.type=single-node --publish 9200:9200 --publish 9300:9300
+      elasticsearch8:
+        image: elastic/elasticsearch:8.5.3
+        options: >-
+          --env discovery.type=single-node
+          --env xpack.security.enabled=false
+          --env xpack.security.http.ssl.enabled=false
+          --env action.destructive_requires_name=false
+          --env cluster.routing.allocation.disk.threshold_enabled=false
+          --publish 9200:9200
     steps:
     - name: Download and install latest SonarScanner CLI tool
       run: |
@@ -167,9 +173,6 @@ jobs:
         df -h
         echo "Cleanup"
         rm -rf /tmp/graphene*
-        curl -XPUT -H "Content-Type: application/json" http://localhost:9200/_cluster/settings \
-          -d '{ "transient": { "cluster.routing.allocation.disk.threshold_enabled": false } }'
-        echo
         _build/tests/es_test -l test_suite
         df -h
         echo "Cleanup"

libraries/plugins/elasticsearch/elasticsearch_plugin.cpp

@@ -630,7 +630,8 @@ operation_history_object elasticsearch_plugin::get_operation_by_id( const operat
    }
    )";
 
-   const auto response = my->es->query( my->_options.index_prefix + "*/_doc/_search", query );
+   const auto uri = my->_options.index_prefix + ( my->is_es_version_7_or_above ? "*/_search" : "*/_doc/_search" );
+   const auto response = my->es->query( uri, query );
    variant variant_response = fc::json::from_string(response);
    const auto source = variant_response["hits"]["hits"][size_t(0)]["_source"];
    return fromEStoOperation(source);
@@ -677,7 +678,8 @@ vector<operation_history_object> elasticsearch_plugin::get_account_history(
    if( !my->es->check_status() )
       return result;
 
-   const auto response = my->es->query( my->_options.index_prefix + "*/_doc/_search", query );
+   const auto uri = my->_options.index_prefix + ( my->is_es_version_7_or_above ? "*/_search" : "*/_doc/_search" );
+   const auto response = my->es->query( uri, query );
 
    variant variant_response = fc::json::from_string(response);
 

libraries/utilities/elasticsearch.cpp

@@ -211,13 +211,14 @@ std::string es_client::get_version() const
 
    fc::variant content = fc::json::from_string( response.content );
    return content["version"]["number"].as_string();
-} FC_CAPTURE_AND_RETHROW() }
+} FC_CAPTURE_LOG_AND_RETHROW( (base_url) ) }
 
 void es_client::check_version_7_or_above( bool& result ) const noexcept
 {
    static const int64_t version_7 = 7;
    try {
       const auto es_version = get_version();
+      ilog( "ES version detected: ${v}", ("v", es_version) );
       auto dot_pos = es_version.find('.');
       result = ( std::stoi(es_version.substr(0,dot_pos)) >= version_7 );
    }

tests/common/database_fixture.cpp

@@ -407,7 +407,6 @@ std::shared_ptr<boost::program_options::variables_map> database_fixture_base::in
    }
    // load ES or AH, but not both
    if(fixture.current_test_name == "elasticsearch_account_history" ||
-         fixture.current_test_name == "elasticsearch_suite" ||
          fixture.current_test_name == "elasticsearch_history_api") {
       fixture.app.register_plugin<graphene::elasticsearch::elasticsearch_plugin>(true);
 
@@ -429,7 +428,7 @@ std::shared_ptr<boost::program_options::variables_map> database_fixture_base::in
       fixture.app.register_plugin<graphene::account_history::account_history_plugin>(true);
    }
 
-   if(fixture.current_test_name == "elasticsearch_objects" || fixture.current_test_name == "elasticsearch_suite") {
+   if( fixture.current_test_name == "elasticsearch_objects" ) {
       fixture.app.register_plugin<graphene::es_objects::es_objects_plugin>(true);
 
       fc::set_option( options, "es-objects-elasticsearch-url", GRAPHENE_TESTING_ES_URL );

tests/common/elasticsearch.cpp

@@ -48,7 +48,6 @@ bool checkES(ES& es)
    if(doCurl(curl_request).empty())
       return false;
    return true;
-
 }
 
 std::string getESVersion(ES& es)
@@ -69,6 +68,7 @@ void checkESVersion7OrAbove(ES& es, bool& result) noexcept
    static const int64_t version_7 = 7;
    try {
       const auto es_version = graphene::utilities::getESVersion(es);
+      ilog( "ES version detected: ${v}", ("v", es_version) );
       auto dot_pos = es_version.find('.');
       result = ( std::stoi(es_version.substr(0,dot_pos)) >= version_7 );
    }
@@ -100,10 +100,29 @@ bool deleteAll(ES& es)
    curl_request.type = "DELETE";
 
    auto curl_response = doCurl(curl_request);
-   if(curl_response.empty())
+   if( curl_response.empty() )
+   {
+      wlog( "Empty ES response" );
+      return false;
+   }
+
+   // Check errors in response
+   try
+   {
+      fc::variant j = fc::json::from_string(curl_response);
+      if( j.is_object() && j.get_object().contains("error") )
+      {
+         wlog( "ES returned an error: ${r}", ("r", curl_response) );
+         return false;
+      }
+   }
+   catch( const fc::exception& e )
+   {
+      wlog( "Error while checking ES response ${r}", ("r", curl_response) );
+      wdump( (e.to_detail_string()) );
       return false;
-   else
-      return true;
+   }
+   return true;
 }
 
 std::string getEndPoint(ES& es)
@@ -147,6 +166,12 @@ std::string doCurl(CurlRequest& curl)
       curl_easy_setopt(curl.handler, CURLOPT_USERPWD, curl.auth.c_str());
    curl_easy_perform(curl.handler);
 
+   long code;
+   curl_easy_getinfo( curl.handler, CURLINFO_RESPONSE_CODE, &code );
+
+   if( 200 != code )
+      wlog( "doCurl response [${code}] ${msg}", ("code", ((int64_t)code))("msg", CurlReadBuffer) );
+
    return CurlReadBuffer;
 }
 

tests/elasticsearch/main.cpp

@@ -71,7 +71,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
          generate_block();
 
          string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }";
-         es.endpoint = es.index_prefix + "*/_doc/_count";
+         es.endpoint = es.index_prefix + "*/_count";
          es.query = query;
 
          string res;
@@ -85,7 +85,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
             return (total == "5");
          });
 
-         es.endpoint = es.index_prefix + "*/_doc/_search";
+         es.endpoint = es.index_prefix + "*/_search";
          res = graphene::utilities::simpleQuery(es);
          j = fc::json::from_string(res);
          auto first_id = j["hits"]["hits"][size_t(0)]["_id"].as_string();
@@ -95,7 +95,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
          auto willie = create_account("willie");
          generate_block();
 
-         es.endpoint = es.index_prefix + "*/_doc/_count";
+         es.endpoint = es.index_prefix + "*/_count";
 
          fc::wait_for( ES_WAIT_TIME,  [&]() {
             res = graphene::utilities::simpleQuery(es);
@@ -197,7 +197,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_account_history) {
 
          generate_block();
 
-         es.endpoint = es.index_prefix + "*/_doc/_count";
+         es.endpoint = es.index_prefix + "*/_count";
          fc::wait_for( ES_WAIT_TIME,  [&]() {
             res = graphene::utilities::simpleQuery(es);
             j = fc::json::from_string(res);
@@ -241,7 +241,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
          generate_block();
 
          string query = "{ \"query\" : { \"bool\" : { \"must\" : [{\"match_all\": {}}] } } }";
-         es.endpoint = es.index_prefix + "*/_doc/_count";
+         es.endpoint = es.index_prefix + "*/_count";
          es.query = query;
 
          string res;
@@ -255,14 +255,14 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
             return (total == "2");
          });
 
-         es.endpoint = es.index_prefix + "asset/_doc/_search";
+         es.endpoint = es.index_prefix + "asset/_search";
          res = graphene::utilities::simpleQuery(es);
          j = fc::json::from_string(res);
          auto first_id = j["hits"]["hits"][size_t(0)]["_source"]["symbol"].as_string();
          BOOST_CHECK_EQUAL(first_id, "USD");
 
          auto bitasset_data_id = j["hits"]["hits"][size_t(0)]["_source"]["bitasset_data_id"].as_string();
-         es.endpoint = es.index_prefix + "bitasset/_doc/_search";
+         es.endpoint = es.index_prefix + "bitasset/_search";
          es.query = "{ \"query\" : { \"bool\": { \"must\" : [{ \"term\": { \"object_id\": \""
                   + bitasset_data_id + "\"}}] } } }";
          res = graphene::utilities::simpleQuery(es);
@@ -275,7 +275,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
                             db.get_dynamic_global_properties().next_maintenance_time );
          generate_block();
 
-         es.endpoint = es.index_prefix + "limitorder/_doc/_count";
+         es.endpoint = es.index_prefix + "limitorder/_count";
          es.query = "";
          fc::wait_for( ES_WAIT_TIME,  [&]() {
             res = graphene::utilities::getEndPoint(es);
@@ -293,7 +293,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
          generate_blocks( db.get_dynamic_global_properties().next_maintenance_time );
          generate_block();
 
-         es.endpoint = es.index_prefix + "budget/_doc/_count";
+         es.endpoint = es.index_prefix + "budget/_count";
          es.query = "";
          fc::wait_for( ES_WAIT_TIME,  [&]() {
             res = graphene::utilities::getEndPoint(es);
@@ -307,7 +307,7 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
             return (total == "1"); // new record inserted at the first maintenance block
          });
 
-         es.endpoint = es.index_prefix + "limitorder/_doc/_count";
+         es.endpoint = es.index_prefix + "limitorder/_count";
          es.query = "";
          fc::wait_for( ES_WAIT_TIME,  [&]() {
             res = graphene::utilities::getEndPoint(es);
@@ -329,39 +329,6 @@ BOOST_AUTO_TEST_CASE(elasticsearch_objects) {
    }
 }
 
-BOOST_AUTO_TEST_CASE(elasticsearch_suite) {
-   try {
-
-      CURL *curl; // curl handler
-      curl = curl_easy_init();
-      curl_easy_setopt(curl, CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1_2);
-
-      graphene::utilities::ES es;
-      es.curl = curl;
-      es.elasticsearch_url = GRAPHENE_TESTING_ES_URL;
-      es.index_prefix = es_index_prefix;
-      auto delete_account_history = graphene::utilities::deleteAll(es);
-      BOOST_REQUIRE(delete_account_history); // require successful deletion
-
-      graphene::utilities::ES es_obj;
-      es_obj.curl = curl;
-      es_obj.elasticsearch_url = GRAPHENE_TESTING_ES_URL;
-      es_obj.index_prefix = es_obj_index_prefix;
-      auto delete_objects = graphene::utilities::deleteAll(es_obj);
-      BOOST_REQUIRE(delete_objects); // require successful deletion
-
-      if(delete_account_history && delete_objects) { // all records deleted
-
-
-      }
-      // Note: this test case ends too quickly, sometimes causing an memory access violation on cleanup
-   }
-   catch (fc::exception &e) {
-      edump((e.to_detail_string()));
-      throw;
-   }
-}
-
 BOOST_AUTO_TEST_CASE(elasticsearch_history_api) {
    try {
       CURL *curl; // curl handler
@@ -390,8 +357,8 @@ BOOST_AUTO_TEST_CASE(elasticsearch_history_api) {
 
          generate_block();
 
+         // Test history APIs
          graphene::app::history_api hist_api(app);
-         app.enable_plugin("elasticsearch");
 
          // f(A, 0, 4, 9) = { 5, 3, 1, 0 }
          auto histories = hist_api.get_account_history(
@@ -685,6 +652,16 @@ BOOST_AUTO_TEST_CASE(elasticsearch_history_api) {
          BOOST_CHECK_EQUAL(histories[2].id.instance(), 3u);
          BOOST_CHECK_EQUAL(histories[3].id.instance(), 1u);
          BOOST_CHECK_EQUAL(histories[4].id.instance(), 0u);
+
+         // Ugly test to cover elasticsearch_plugin::get_operation_by_id()
+         if( !app.elasticsearch_thread )
+            app.elasticsearch_thread = std::make_shared<fc::thread>("elasticsearch");
+         auto es_plugin = app.get_plugin< graphene::elasticsearch::elasticsearch_plugin >("elasticsearch");
+         auto his_obj7 = app.elasticsearch_thread->async([&es_plugin]() {
+            return es_plugin->get_operation_by_id( operation_history_id_type(7) );
+         }, "thread invoke for method " BOOST_PP_STRINGIZE(method_name)).wait();
+         BOOST_REQUIRE( his_obj7.op.is_type<account_create_operation>() );
+         BOOST_CHECK_EQUAL( his_obj7.op.get<account_create_operation>().name, "alice" );
       }
    }
    catch (fc::exception &e) {