HTTP Error 411 while making POST request to Microsoft Azure Workspace

[prat33kg]

I am trying to set up a remote log system on the Microsoft Azure Workspace. I am using cpr POST request to send sample JSON data over to the server. The code I used is as follows:

#include <iostream>
#include <map>
#include <utility>
#include <nlohmann/json.hpp>
using json = nlohmann::json;
#include <cpr/cpr.h>
#include <time.h>
using std::cout;
using std::cerr;
using std::endl;

#include <string>
using std::string;

#include "cryptopp/cryptlib.h"
using CryptoPP::Exception;

#include "cryptopp/hmac.h"
using CryptoPP::HMAC;

#include "cryptopp/sha.h"
using CryptoPP::SHA256;

#include "cryptopp/base64.h"
using CryptoPP::Base64Encoder;
using CryptoPP::Base64Decoder;

#include "cryptopp/filters.h"
using CryptoPP::StringSink;
using CryptoPP::StringSource;
using CryptoPP::HashFilter;



typedef unsigned char byte;

string CUSTOMER_ID = "xxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx";
string SHARED_KEY = "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx";
string METHOD = "POST";
string CONTENT_TYPE = "application/json";
string RESOURCE = "/api/logs";
string uri = "https://" + CUSTOMER_ID + ".ods.opinsights.azure.com/api/logs?api-version=2016-04-01";
string uri2 = "https://postman-echo.com/post";

string sign(string key, string plain)
{
    string mac, encoded;
    try
    {

        HMAC< SHA256 > hmac((byte*)key.c_str(), key.length());

        StringSource(plain, true,
                     new HashFilter(hmac,
                                    new StringSink(mac)
                     ) // HashFilter
        ); // StringSource
    }
    catch(const CryptoPP::Exception& e)
    {
        cerr << e.what() << endl;
    }

    encoded.clear();
    StringSource(mac, true,
                 new Base64Encoder(
                         new StringSink(encoded)
                 ) // Base64Encoder
    ); // StringSource

    return encoded;
}

string getTimestamp()
{
    char buf[1000];
    time_t now = time(0);
    //struct tm tm = *localtime(&now);
    struct tm *tm = gmtime(&now);
    strftime(buf, sizeof buf, "%a, %d %b %Y %H:%M:%S GMT", tm);
    cout << buf << endl;
    return buf;

}

string getSignature(const string& customer_id, const string& shared_key, const string& date, size_t content_length,
                    const string& method, const string& content_type, const string& resource)
{
    string x_headers = "x-ms-date:" + date;
    string string_to_hash = (method + "\n" + std::to_string(content_length) + "\n" + content_type + "\n" + x_headers + "\n" + resource);
    string decoded_share_key;
    decoded_share_key.clear();
    StringSource(shared_key,true, new Base64Decoder( new StringSink(decoded_share_key)));
    string encoded_hash = sign(decoded_share_key,string_to_hash);

    string signature = "SharedKey "+ customer_id + ":" +encoded_hash;
            //SharedKey {customer_id}:{encoded_hash}

    cout << signature << endl;

    return signature;
}


void post(string customer_id, string shared_key, string body, string log_type)
{
    string date = getTimestamp();

    string signature = getSignature(CUSTOMER_ID, SHARED_KEY, date,
                                    body.length(), METHOD,
                                    CONTENT_TYPE, RESOURCE);

    auto response = cpr::Post(cpr::Url(uri),
                                cpr::Header{{"content-type",CONTENT_TYPE},
                                            {"Authorization",signature},
                                            {"log-type",log_type},
                                            {"x-ms-date", date},
                                            {"content-length",std::to_string(body.length())}},
                                cpr::Body(body));



    cout << response.url << endl;
    try {
        json j = json::parse(response.text);
        cout << j.dump(4) << endl;
    }
    catch (std::exception& e) {
        cout << e.what() << endl << endl;
        cout << response.text << endl;
    }
    cout << response.status_code << endl;
    if(response.status_code!=200)
    {
       cout << response.error.message << endl;
    }
    cout << response.elapsed << endl;
}


int main() {
    
    string log_type = "LoggingTest";

    json log_msg;
    json msg;
    log_msg["property1"] = "value1";

    msg = {log_msg};

    post(CUSTOMER_ID, SHARED_KEY, msg.dump(), log_type);


    return 0;
}

However, I'm always getting the same error like this (from response.text) :

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN""http://www.w3.org/TR/html4/strict.dtd">
<HTML><HEAD><TITLE>Length Required</TITLE>
<META HTTP-EQUIV="Content-Type" Content="text/html; charset=us-ascii"></HEAD>
<BODY><h2>Length Required</h2>
<hr><p>HTTP Error 411. The request must be chunked or have a content length.</p>
</BODY></HTML>

Things I have tried and the results:

1. Added "content-length" header: Same error
2. Tried sending empty body with content length zero: Same error
3. Made POST request to Postman-echo service to see if the library is unable to forward all data: Results as follows -

3.1 Sending JSON data with authorization header:

{
    "args": {},
    "data": {},
    "files": {},
    "form": {},
    "headers": {
        "accept": "*/*",
        "authorization": "SharedKey xxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
        "host": "postman-echo.com",
        "user-agent": "curl/7.47.0",
        "x-amzn-trace-id": "Root=1-5ebe2da5-2275dd11f1b39c54c33fa6df",
        "x-forwarded-port": "443",
        "x-forwarded-proto": "https"
    },
    "json": null,
    "url": "https://postman-echo.com/post"
}

Notice that even though I'm sending JSON data with the POST request, Postman-echo does not show it. This is strange because when I did the same thing using a Python script, the result showed me both the data as well as the authorization signature.

3.2 Sending JSON data without authorization header:

{
     "args": {},
    "data": [
        {
            "property1": "value1"
        }
    ],
    "files": {},
    "form": {},
    "headers": {
        "accept": "*/*",
        "content-length": "24",
        "content-type": "application/json",
        "host": "postman-echo.com",
        "log-type": "LoggingTest",
        "user-agent": "curl/7.47.0",
        "x-amzn-trace-id": "Root=1-5ebe2f54-bfcef7b08ee37ca8efe47d90",
        "x-forwarded-port": "443",
        "x-forwarded-proto": "https",
        "x-ms-date": "Fri, 15 May 2020 05:57:40 GMT"
    },
    "json": [
        {
            "property1": "value1"
        }
    ],
    "url": "https://postman-echo.com/post"
}

Then Postman-echo is showing me the json data sent.

In my understanding, generated key is not wrong, since in that case, authorization failure is the first error. The same code above, written in Python3 and Python2 worked correctly. This was the link I was following: https://docs.microsoft.com/de-de/azure/azure-monitor/platform/data-collector-api

Is there any known issue of similar kind in the cpr library? Please help.

[COM8]

Arg...
I'm currently in the process of rewriting all unit tests and updating CPR to curl 7.56 (#364).
I think I also noticed this strange behaviour.
I will have a look at it.

Thanks for reporting this.

[prat33kg]

Hey @COM8. Thanks for the response. Could you give an estimated timeline as till when this issue might be resolved? This library being so simple to use, has become important for our project. An estimate would help me plan the project further. Thanks.

[COM8]

@PatronusCoder I would say by the end of next week.
I'm currently finishing up #364 .
After that I will work on a fix for it.

[COM8]

@PatronusCoder I think I already fixed this issue with the following commit to #364 :
050a46e

[COM8]

For validation could you please come up with a simple test case, where this error occurs, please?
Like it's done here: https://github.com/whoshuu/cpr/blob/master/test/post_tests.cpp

This would help a lot!

[prat33kg]

I'll try it today and definitely try to work out a simple test case. Thanks!

[prat33kg]

Hey @COM8. That commit did not solve the issue.

[prat33kg]

Hey @COM8. Found another thing interesting which might be helpful. When I pass the authorization signature as an lvalue, then the Postman-echo service does not show the body. However, if I pass it as an rvalue (generated a key for a future time and then made post request), it worked fine.

[COM8]

Hey @COM8. Found another thing interesting which might be helpful. When I pass the authorization signature as an lvalue, then the Postman-echo service does not show the body. However, if I pass it as an rvalue (generated a key for a future time and then made post request), it worked fine.

@PatronusCoder this could be an interesting one since Header is just a:

using Header = std::map<std::string, std::string, CaseInsensitiveCompare>;

[COM8]

As soon as #276 and #364 have been merged into master, I will have time to investigate those.

[COM8]

In your example from above, you are never setting the json msg to anything when calling post(...)?
Were you able to create a minimal example where this issue occurred?

[prat33kg]

Hey @COM8. I made the change in the code above. It was typo so thanks for that. Also, I couldn't generate a minimal example. The authorization key signature is dynamically generated and that is where it doesn't work. Therefore I could not make a minimal example for this issue. The code above was actually built as a minimal example for testing.

[COM8]

I'm still unable to reproduce this bug.
I created a new simplified test case based on your given program and it works just fine:

static std::string getTimestamp() {
    char buf[1000];
    time_t now = time(0);
    struct tm* tm = gmtime(&now);
    strftime(buf, sizeof buf, "%a, %d %b %Y %H:%M:%S GMT", tm);
    return buf;
}

TEST(UrlEncodedPostTests, PostReflectTest) {
    std::string uri = server->GetBaseUrl() + "/reflect_post.html";
    std::string body = "{\"property1\": \"value1\"}";
    std::string contentType = "application/json";
    std::string signature = "x-ms-date: something";
    std::string logType = "LoggingTest";
    std::string date = getTimestamp();
    auto response = cpr::Post(cpr::Url(uri),
                              cpr::Header{{"content-type", contentType},
                                          {"Authorization", signature},
                                          {"log-type", logType},
                                          {"x-ms-date", date},
                                          {"content-length", std::to_string(body.length())}},
                              cpr::Body(body));
    EXPECT_EQ(ErrorCode::OK, response.error.code);
    EXPECT_EQ(200, response.status_code);
    EXPECT_EQ(body, response.text);
    EXPECT_EQ(std::string{"application/json"}, response.header["content-type"]);
    EXPECT_EQ(signature, response.header["Authorization"]);
    EXPECT_EQ(logType, response.header["log-type"]);
    EXPECT_EQ(date, response.header["x-ms-date"]);
    EXPECT_EQ(std::to_string(body.length()), response.header["content-length"]);
}

So I don't this this is a fault of CPR.
Please, try to isolate the broken behaviour, because else I'm unable to reproduce it.
You have to provide a simple example test case I can run to check against for me to be able to fix this bug.
Perhaps one without all your dependencies. One with just CPR so we can be sure it's caused by CPR.

[prat33kg]

Found the problem. The getSignature() function in my code above (probably when the signature is created by the encodings), appends a new line character at the end of the string. If I give this string as a value to the header key, then the body is not appended to the post request made to Postman-echo service. If its not already know, maybe you can add it as a check in the code or as a remark in the documentation. Thanks @COM8 for being patient and helpful.