Skip to content

Commit

Permalink
Merge pull request #863 from AudricV/add-content-type-and-content-len…
Browse files Browse the repository at this point in the history 
…gth-headers-to-post-requests

Add Content-Type header to all POST requests without an empty body
  • Loading branch information
@AudricV
AudricV authored Dec 16, 2022
2 parents 88e07e5 + 34d79bd commit d5437e0
Show file tree
Hide file tree
Showing 307 changed files with 9,302 additions and 9,222 deletions.
101 changes: 96 additions & 5 deletions extractor/src/main/java/org/schabi/newpipe/extractor/downloader/Downloader.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
import java.io.IOException;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

Expand Down Expand Up @@ -39,7 +41,7 @@ public Response get(final String url) throws IOException, ReCaptchaException {
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
*/
public Response get(final String url, @Nullable final Localization localization)
public Response get(final String url, final Localization localization)
throws IOException, ReCaptchaException {
return get(url, null, localization);
}
Expand Down Expand Up @@ -70,7 +72,7 @@ public Response get(final String url, @Nullable final Map<String, List<String>>
*/
public Response get(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final Localization localization)
final Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
.get(url)
Expand Down Expand Up @@ -112,7 +114,7 @@ public Response head(final String url, @Nullable final Map<String, List<String>>
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the GET request
* @return the result of the POST request
*/
public Response post(final String url,
@Nullable final Map<String, List<String>> headers,
Expand All @@ -131,12 +133,12 @@ public Response post(final String url,
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the GET request
* @return the result of the POST request
*/
public Response post(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
@Nullable final Localization localization)
final Localization localization)
throws IOException, ReCaptchaException {
return execute(Request.newBuilder()
.post(url, dataToSend)
Expand All @@ -145,6 +147,95 @@ public Response post(final String url,
.build());
}

/**
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header with a given {@link Localization}.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @param contentType the mime type of the body sent, which will be set as the value of the
* {@code Content-Type} header
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization,
final String contentType)
throws IOException, ReCaptchaException {
final Map<String, List<String>> actualHeaders = new HashMap<>();
if (headers != null) {
actualHeaders.putAll(headers);
}
actualHeaders.put("Content-Type", Collections.singletonList(contentType));
return post(url, actualHeaders, dataToSend, localization);
}

/**
* Convenient method to send a POST request using the specified value of the
* {@code Content-Type} header.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param contentType the mime type of the body sent, which will be set as the value of the
* {@code Content-Type} header
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentType(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final String contentType)
throws IOException, ReCaptchaException {
return postWithContentType(url, headers, dataToSend, NewPipe.getPreferredLocalization(),
contentType);
}

/**
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header with a given {@link Localization}.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @param localization the source of the value of the {@code Accept-Language} header
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend,
final Localization localization)
throws IOException, ReCaptchaException {
return postWithContentType(url, headers, dataToSend, localization, "application/json");
}

/**
* Convenient method to send a POST request the JSON mime type as the value of the
* {@code Content-Type} header.
*
* @param url the URL that is pointing to the wanted resource
* @param headers a list of headers that will be used in the request.
* Any default headers <b>should</b> be overridden by these.
* @param dataToSend byte array that will be sent when doing the request.
* @return the result of the POST request
* @see #post(String, Map, byte[], Localization)
*/
public Response postWithContentTypeJson(final String url,
@Nullable final Map<String, List<String>> headers,
@Nullable final byte[] dataToSend)
throws IOException, ReCaptchaException {
return postWithContentTypeJson(url, headers, dataToSend,
NewPipe.getPreferredLocalization());
}

/**
* Do a request using the specified {@link Request} object.
*
Expand Down
35 changes: 16 additions & 19 deletions ...va/org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampExtractorHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,22 +6,23 @@
import com.grack.nanojson.JsonParser;
import com.grack.nanojson.JsonParserException;
import com.grack.nanojson.JsonWriter;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Element;
import org.schabi.newpipe.extractor.NewPipe;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.exceptions.ReCaptchaException;
import org.schabi.newpipe.extractor.localization.DateWrapper;
import org.schabi.newpipe.extractor.utils.Utils;

import javax.annotation.Nullable;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.time.DateTimeException;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
import java.util.Locale;

import javax.annotation.Nullable;

public final class BandcampExtractorHelper {

public static final String BASE_URL = "https://bandcamp.com";
Expand All @@ -43,8 +44,8 @@ public static String getStreamUrlFromIds(final long bandId,
+ "&tralbum_id=" + itemId + "&tralbum_type=" + itemType.charAt(0))
.responseBody();

return JsonParser.object().from(jsonString)
.getString("bandcamp_url").replace("http://", "https://");
return Utils.replaceHttpWithHttps(JsonParser.object().from(jsonString)
.getString("bandcamp_url"));

} catch (final JsonParserException | ReCaptchaException | IOException e) {
throw new ParsingException("Ids could not be translated to URL", e);
Expand All @@ -60,19 +61,15 @@ public static String getStreamUrlFromIds(final long bandId,
*/
public static JsonObject getArtistDetails(final String id) throws ParsingException {
try {
return
JsonParser.object().from(
NewPipe.getDownloader().post(
BASE_API_URL + "/mobile/22/band_details",
null,
JsonWriter.string()
.object()
.value("band_id", id)
.end()
.done()
.getBytes()
).responseBody()
);
return JsonParser.object().from(NewPipe.getDownloader().postWithContentTypeJson(
BASE_API_URL + "/mobile/22/band_details",
Collections.emptyMap(),
JsonWriter.string()
.object()
.value("band_id", id)
.end()
.done()
.getBytes(StandardCharsets.UTF_8)).responseBody());
} catch (final IOException | ReCaptchaException | JsonParserException e) {
throw new ParsingException("Could not download band details", e);
}
Expand Down Expand Up @@ -123,7 +120,7 @@ public static boolean isSupportedDomain(final String url) throws ParsingExceptio
/**
* Whether the URL points to a radio kiosk.
* @param url the URL to check
* @return true if the URL matches <code>https://bandcamp.com/?show=SHOW_ID</code>
* @return true if the URL matches {@code https://bandcamp.com/?show=SHOW_ID}
*/
public static boolean isRadioUrl(final String url) {
return url.toLowerCase().matches("https?://bandcamp\\.com/\\?show=\\d+");
Expand Down
12 changes: 7 additions & 5 deletions .../org/schabi/newpipe/extractor/services/bandcamp/extractors/BandcampFeaturedExtractor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import javax.annotation.Nonnull;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Collections;

import static org.schabi.newpipe.extractor.services.bandcamp.extractors.BandcampExtractorHelper.BASE_API_URL;

Expand All @@ -40,11 +42,11 @@ public BandcampFeaturedExtractor(final StreamingService streamingService,
public void onFetchPage(@Nonnull final Downloader downloader)
throws IOException, ExtractionException {
try {
json = JsonParser.object().from(
getDownloader().post(
FEATURED_API_URL, null, "{\"platform\":\"\",\"version\":0}".getBytes()
).responseBody()
);
json = JsonParser.object().from(getDownloader().postWithContentTypeJson(
FEATURED_API_URL,
Collections.emptyMap(),
"{\"platform\":\"\",\"version\":0}".getBytes(StandardCharsets.UTF_8))
.responseBody());
} catch (final JsonParserException e) {
throw new ParsingException("Could not parse Bandcamp featured API response", e);
}
Expand Down
49 changes: 27 additions & 22 deletions ...tor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,13 +556,13 @@ public static boolean areHardcodedClientVersionAndKeyValid()

final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", singletonList("1"));
headers.put("X-YouTube-Client-Version",
singletonList(HARDCODED_CLIENT_VERSION));
headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_CLIENT_VERSION));

// This endpoint is fetched by the YouTube website to get the items of its main menu and is
// pretty lightweight (around 30kB)
final Response response = getDownloader().post(YOUTUBEI_V1_URL + "guide?key="
+ HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER, headers, body);
final Response response = getDownloader().postWithContentTypeJson(
YOUTUBEI_V1_URL + "guide?key=" + HARDCODED_KEY + DISABLE_PRETTY_PRINT_PARAMETER,
headers, body);
final String responseBody = response.responseBody();
final int responseCode = response.responseCode();

Expand Down Expand Up @@ -800,15 +800,12 @@ public static boolean isHardcodedYoutubeMusicKeyValid() throws IOException,
// @formatter:on

final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", singletonList(
HARDCODED_YOUTUBE_MUSIC_KEY[1]));
headers.put("X-YouTube-Client-Version", singletonList(
HARDCODED_YOUTUBE_MUSIC_KEY[2]));
headers.put("X-YouTube-Client-Name", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[1]));
headers.put("X-YouTube-Client-Version", singletonList(HARDCODED_YOUTUBE_MUSIC_KEY[2]));
headers.put("Origin", singletonList("https://music.youtube.com"));
headers.put("Referer", singletonList("music.youtube.com"));
headers.put("Content-Type", singletonList("application/json"));
headers.put("Referer", singletonList("https://music.youtube.com"));

final Response response = getDownloader().post(url, headers, json);
final Response response = getDownloader().postWithContentTypeJson(url, headers, json);
// Ensure to have a valid response
return response.responseBody().length() > 500 && response.responseCode() == 200;
}
Expand Down Expand Up @@ -1180,13 +1177,11 @@ public static JsonObject getJsonPostResponse(final String endpoint,
final Localization localization)
throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
headers.put("Content-Type", singletonList("application/json"));

final Response response = getDownloader().post(YOUTUBEI_V1_URL + endpoint + "?key="
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization);
addYouTubeHeaders(headers);

return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(YOUTUBEI_V1_URL + endpoint + "?key="
+ getKey() + DISABLE_PRETTY_PRINT_PARAMETER, headers, body, localization)));
}

public static JsonObject getJsonAndroidPostResponse(
Expand Down Expand Up @@ -1215,17 +1210,17 @@ private static JsonObject getMobilePostResponse(
@Nonnull final String innerTubeApiKey,
@Nullable final String endPartOfUrlRequest) throws IOException, ExtractionException {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("Content-Type", singletonList("application/json"));
headers.put("User-Agent", singletonList(userAgent));
headers.put("X-Goog-Api-Format-Version", singletonList("2"));

final String baseEndpointUrl = YOUTUBEI_V1_GAPIS_URL + endpoint + "?key=" + innerTubeApiKey
+ DISABLE_PRETTY_PRINT_PARAMETER;

final Response response = getDownloader().post(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl : baseEndpointUrl + endPartOfUrlRequest,
headers, body, localization);
return JsonUtils.toJsonObject(getValidJsonResponseBody(response));
return JsonUtils.toJsonObject(getValidJsonResponseBody(
getDownloader().postWithContentTypeJson(isNullOrEmpty(endPartOfUrlRequest)
? baseEndpointUrl
: baseEndpointUrl + endPartOfUrlRequest,
headers, body, localization)));
}

@Nonnull
Expand Down Expand Up @@ -1428,6 +1423,16 @@ public static String getIosUserAgent(@Nullable final Localization localization)
+ ")";
}

@Nonnull
public static Map<String, List<String>> getYoutubeMusicHeaders() {
final Map<String, List<String>> headers = new HashMap<>();
headers.put("X-YouTube-Client-Name", Collections.singletonList(youtubeMusicKey[1]));
headers.put("X-YouTube-Client-Version", Collections.singletonList(youtubeMusicKey[2]));
headers.put("Origin", Collections.singletonList("https://music.youtube.com"));
headers.put("Referer", Collections.singletonList("https://music.youtube.com"));
return headers;
}

/**
* Add required headers and cookies to an existing headers Map.
* @see #addClientInfoHeaders(Map)
Expand Down
4 changes: 2 additions & 2 deletions ...ipe/extractor/services/youtube/dashmanifestcreators/YoutubeDashManifestCreatorsUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
import java.util.Map;
import java.util.Objects;

import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.addClientInfoHeaders;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getAndroidUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.getIosUserAgent;
import static org.schabi.newpipe.extractor.services.youtube.YoutubeParsingHelper.isAndroidStreamingUrl;
Expand Down Expand Up @@ -707,7 +706,8 @@ private static Response getStreamingWebUrlWithoutRedirects(
throws CreationException {
try {
final Map<String, List<String>> headers = new HashMap<>();
addClientInfoHeaders(headers);
headers.put("Origin", Collections.singletonList("https://www.youtube.com"));
headers.put("Referer", Collections.singletonList("https://www.youtube.com"));

String responseMimeType = "";

Expand Down
Loading

0 comments on commit d5437e0

Please sign in to comment.