0
0
mirror of https://github.com/TeamNewPipe/NewPipeExtractor.git synced 2024-09-19 19:42:28 +02:00

Use the new URL encode/decode methods introduced in Java 10 (#1196)

* Use Java 10 URLDecoder/URLEncoder methods

* Simplify compatParseMap
This commit is contained in:
Isira Seneviratne 2024-07-21 19:45:28 +05:30 committed by GitHub
parent 592f1596e6
commit 9a29f9ee2d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 69 additions and 150 deletions

View File

@ -21,7 +21,7 @@ If you're using Gradle, you could add NewPipe Extractor as a dependency with the
-dontwarn org.mozilla.javascript.tools.**
```
**Note:** To use NewPipe Extractor in Android projects with a `minSdk` below 26, [API desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) is required. If the `minSdk` is below 19, the `desugar_jdk_libs_nio` artifact is required, which requires Android Gradle Plugin (AGP) version 7.4.0.
**Note:** To use NewPipe Extractor in Android projects with a `minSdk` below 33, [core library desugaring](https://developer.android.com/studio/write/java8-support#library-desugaring) with the `desugar_jdk_libs_nio` artifact is required.
### Testing changes

View File

@ -8,7 +8,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.UnsupportedEncodingException;
import java.util.List;
public final class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@ -28,10 +27,6 @@ public final class BandcampSearchQueryHandlerFactory extends SearchQueryHandlerF
final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("query \"" + query + "\" could not be encoded", e);
}
return BASE_URL + "/search?q=" + Utils.encodeUrlUtf8(query) + "&page=1";
}
}

View File

@ -4,7 +4,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.UnsupportedEncodingException;
import java.util.List;
public final class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@ -42,10 +41,6 @@ public final class MediaCCCSearchQueryHandlerFactory extends SearchQueryHandlerF
final List<String> contentFilter,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not create search string with query: " + query, e);
}
return "https://media.ccc.de/public/events/search?q=" + Utils.encodeUrlUtf8(query);
}
}

View File

@ -38,7 +38,6 @@ import org.schabi.newpipe.extractor.utils.JsonUtils;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -379,8 +378,7 @@ public class PeertubeStreamExtractor extends StreamExtractor {
}
@Nonnull
private String getRelatedItemsUrl(@Nonnull final List<String> tags)
throws UnsupportedEncodingException {
private String getRelatedItemsUrl(@Nonnull final List<String> tags) {
final String url = baseUrl + PeertubeSearchQueryHandlerFactory.SEARCH_ENDPOINT_VIDEOS;
final StringBuilder params = new StringBuilder();
params.append("start=0&count=8&sort=-createdAt");

View File

@ -5,7 +5,6 @@ import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.UnsupportedEncodingException;
import java.util.List;
public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@ -49,21 +48,17 @@ public final class PeertubeSearchQueryHandlerFactory extends SearchQueryHandlerF
final String sortFilter,
final String baseUrl)
throws ParsingException, UnsupportedOperationException {
try {
final String endpoint;
if (contentFilters.isEmpty()
|| contentFilters.get(0).equals(VIDEOS)
|| contentFilters.get(0).equals(SEPIA_VIDEOS)) {
endpoint = SEARCH_ENDPOINT_VIDEOS;
} else if (contentFilters.get(0).equals(CHANNELS)) {
endpoint = SEARCH_ENDPOINT_CHANNELS;
} else {
endpoint = SEARCH_ENDPOINT_PLAYLISTS;
}
return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString);
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
final String endpoint;
if (contentFilters.isEmpty()
|| contentFilters.get(0).equals(VIDEOS)
|| contentFilters.get(0).equals(SEPIA_VIDEOS)) {
endpoint = SEARCH_ENDPOINT_VIDEOS;
} else if (contentFilters.get(0).equals(CHANNELS)) {
endpoint = SEARCH_ENDPOINT_CHANNELS;
} else {
endpoint = SEARCH_ENDPOINT_PLAYLISTS;
}
return baseUrl + endpoint + "?search=" + Utils.encodeUrlUtf8(searchString);
}
@Override

View File

@ -23,7 +23,6 @@ import org.schabi.newpipe.extractor.search.SearchExtractor;
import org.schabi.newpipe.extractor.utils.Parser;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.Collections;
@ -159,7 +158,7 @@ public class SoundcloudSearchExtractor extends SearchExtractor {
private int getOffsetFromUrl(final String url) throws ParsingException {
try {
return Integer.parseInt(Parser.compatParseMap(new URL(url).getQuery()).get("offset"));
} catch (MalformedURLException | UnsupportedEncodingException e) {
} catch (final MalformedURLException e) {
throw new ParsingException("Could not get offset from page URL", e);
}
}

View File

@ -39,7 +39,6 @@ import org.schabi.newpipe.extractor.stream.VideoStream;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@ -317,14 +316,6 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
}
}
private static String urlEncode(final String value) {
try {
return Utils.encodeUrlUtf8(value);
} catch (final UnsupportedEncodingException e) {
throw new IllegalStateException(e);
}
}
@Override
public List<VideoStream> getVideoStreams() {
return Collections.emptyList();
@ -344,9 +335,8 @@ public class SoundcloudStreamExtractor extends StreamExtractor {
@Override
public StreamInfoItemsCollector getRelatedItems() throws IOException, ExtractionException {
final StreamInfoItemsCollector collector = new StreamInfoItemsCollector(getServiceId());
final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + urlEncode(getId())
+ "/related?client_id=" + urlEncode(clientId());
final String apiUrl = SOUNDCLOUD_API_V2_URL + "tracks/" + Utils.encodeUrlUtf8(getId())
+ "/related?client_id=" + Utils.encodeUrlUtf8(clientId());
SoundcloudParsingHelper.getStreamsFromApi(collector, apiUrl);
return collector;

View File

@ -10,7 +10,6 @@ import org.schabi.newpipe.extractor.services.soundcloud.SoundcloudParsingHelper;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
public final class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandlerFactory {
@ -60,9 +59,6 @@ public final class SoundcloudSearchQueryHandlerFactory extends SearchQueryHandle
return url + "?q=" + Utils.encodeUrlUtf8(id)
+ "&client_id=" + SoundcloudParsingHelper.clientId()
+ "&limit=" + ITEMS_PER_PAGE + "&offset=0";
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
} catch (final ReCaptchaException e) {
throw new ParsingException("ReCaptcha required", e);
} catch (final IOException | ExtractionException e) {

View File

@ -52,7 +52,6 @@ import org.schabi.newpipe.extractor.utils.RandomStringFromAlphabetGenerator;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
@ -810,11 +809,7 @@ public final class YoutubeParsingHelper {
final String[] params = internUrl.split("&");
for (final String param : params) {
if (param.split("=")[0].equals("q")) {
try {
return Utils.decodeUrlUtf8(param.split("=")[1]);
} catch (final UnsupportedEncodingException e) {
return null;
}
return Utils.decodeUrlUtf8(param.split("=")[1]);
}
}
} else if (internUrl.startsWith("http")) {

View File

@ -96,7 +96,6 @@ import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.stream.Collectors;
@ -1304,7 +1303,7 @@ public class YoutubeStreamExtractor extends StreamExtractor {
return buildAndAddItagInfoToList(videoId, formatData, itagItem,
itagItem.itagType, contentPlaybackNonce);
}
} catch (final IOException | ExtractionException ignored) {
} catch (final ExtractionException ignored) {
// if the itag is not supported and getItag fails, we end up here
}
return null;
@ -1317,19 +1316,18 @@ public class YoutubeStreamExtractor extends StreamExtractor {
@Nonnull final JsonObject formatData,
@Nonnull final ItagItem itagItem,
@Nonnull final ItagItem.ItagType itagType,
@Nonnull final String contentPlaybackNonce) throws IOException, ExtractionException {
@Nonnull final String contentPlaybackNonce) throws ExtractionException {
String streamUrl;
if (formatData.has("url")) {
streamUrl = formatData.getString("url");
} else {
// This url has an obfuscated signature
final String cipherString = formatData.has(CIPHER)
? formatData.getString(CIPHER)
: formatData.getString(SIGNATURE_CIPHER);
final Map<String, String> cipher = Parser.compatParseMap(
cipherString);
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "="
+ YoutubeJavaScriptPlayerManager.deobfuscateSignature(videoId, cipher.get("s"));
final String cipherString = formatData.getString(CIPHER,
formatData.getString(SIGNATURE_CIPHER));
final var cipher = Parser.compatParseMap(cipherString);
final String signature = YoutubeJavaScriptPlayerManager.deobfuscateSignature(videoId,
cipher.getOrDefault("s", ""));
streamUrl = cipher.get("url") + "&" + cipher.get("sp") + "=" + signature;
}
// Add the content playback nonce to the stream URL

View File

@ -6,7 +6,6 @@ import static org.schabi.newpipe.extractor.utils.Utils.isNullOrEmpty;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import org.schabi.newpipe.extractor.linkhandler.SearchQueryHandlerFactory;
import java.io.UnsupportedEncodingException;
import java.util.List;
import javax.annotation.Nonnull;
@ -40,31 +39,22 @@ public final class YoutubeSearchQueryHandlerFactory extends SearchQueryHandlerFa
@Nonnull final List<String> contentFilters,
final String sortFilter)
throws ParsingException, UnsupportedOperationException {
try {
if (!contentFilters.isEmpty()) {
final String contentFilter = contentFilters.get(0);
switch (contentFilter) {
case VIDEOS:
return SEARCH_URL + encodeUrlUtf8(searchString)
+ "&sp=EgIQAfABAQ%253D%253D";
case CHANNELS:
return SEARCH_URL + encodeUrlUtf8(searchString)
+ "&sp=EgIQAvABAQ%253D%253D";
case PLAYLISTS:
return SEARCH_URL + encodeUrlUtf8(searchString)
+ "&sp=EgIQA_ABAQ%253D%253D";
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS:
return MUSIC_SEARCH_URL + encodeUrlUtf8(searchString);
}
}
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=8AEB";
} catch (final UnsupportedEncodingException e) {
throw new ParsingException("Could not encode query", e);
final String contentFilter = !contentFilters.isEmpty() ? contentFilters.get(0) : "";
switch (contentFilter) {
case VIDEOS:
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAfABAQ%253D%253D";
case CHANNELS:
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQAvABAQ%253D%253D";
case PLAYLISTS:
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=EgIQA_ABAQ%253D%253D";
case MUSIC_SONGS:
case MUSIC_VIDEOS:
case MUSIC_ALBUMS:
case MUSIC_PLAYLISTS:
case MUSIC_ARTISTS:
return MUSIC_SEARCH_URL + encodeUrlUtf8(searchString);
default:
return SEARCH_URL + encodeUrlUtf8(searchString) + "&sp=8AEB";
}
}

View File

@ -22,11 +22,11 @@ package org.schabi.newpipe.extractor.utils;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.UnsupportedEncodingException;
import java.util.HashMap;
import java.util.Arrays;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import javax.annotation.Nonnull;
@ -121,17 +121,12 @@ public final class Parser {
}
@Nonnull
public static Map<String, String> compatParseMap(@Nonnull final String input)
throws UnsupportedEncodingException {
final Map<String, String> map = new HashMap<>();
for (final String arg : input.split("&")) {
final String[] splitArg = arg.split("=");
if (splitArg.length > 1) {
map.put(splitArg[0], Utils.decodeUrlUtf8(splitArg[1]));
} else {
map.put(splitArg[0], "");
}
}
return map;
public static Map<String, String> compatParseMap(@Nonnull final String input) {
return Arrays.stream(input.split("&"))
.map(arg -> arg.split("="))
.filter(splitArg -> splitArg.length > 1)
.collect(Collectors.toMap(splitArg -> splitArg[0],
splitArg -> Utils.decodeUrlUtf8(splitArg[1]),
(existing, replacement) -> replacement));
}
}

View File

@ -2,7 +2,6 @@ package org.schabi.newpipe.extractor.utils;
import org.schabi.newpipe.extractor.exceptions.ParsingException;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLDecoder;
@ -33,22 +32,18 @@ public final class Utils {
*
* @param string The string to be encoded.
* @return The encoded URL.
* @throws UnsupportedEncodingException This shouldn't be thrown, as UTF-8 should be supported.
*/
public static String encodeUrlUtf8(final String string) throws UnsupportedEncodingException {
// TODO: Switch to URLEncoder.encode(String, Charset) in Java 10.
return URLEncoder.encode(string, StandardCharsets.UTF_8.name());
public static String encodeUrlUtf8(final String string) {
return URLEncoder.encode(string, StandardCharsets.UTF_8);
}
/**
* Decodes a URL using the UTF-8 character set.
* @param url The URL to be decoded.
* @return The decoded URL.
* @throws UnsupportedEncodingException This shouldn't be thrown, as UTF-8 should be supported.
*/
public static String decodeUrlUtf8(final String url) throws UnsupportedEncodingException {
// TODO: Switch to URLDecoder.decode(String, Charset) in Java 10.
return URLDecoder.decode(url, StandardCharsets.UTF_8.name());
public static String decodeUrlUtf8(final String url) {
return URLDecoder.decode(url, StandardCharsets.UTF_8);
}
/**
@ -155,22 +150,10 @@ public final class Utils {
if (urlQuery != null) {
for (final String param : urlQuery.split("&")) {
final String[] params = param.split("=", 2);
String query;
try {
query = decodeUrlUtf8(params[0]);
} catch (final UnsupportedEncodingException e) {
// Cannot decode string with UTF-8, using the string without decoding
query = params[0];
}
final String query = decodeUrlUtf8(params[0]);
if (query.equals(parameterName)) {
try {
return decodeUrlUtf8(params[1]);
} catch (final UnsupportedEncodingException e) {
// Cannot decode string with UTF-8, using the string without decoding
return params[1];
}
return decodeUrlUtf8(params[1]);
}
}
}

View File

@ -1,6 +1,5 @@
package org.schabi.newpipe.extractor.services.soundcloud.search;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.schabi.newpipe.extractor.ServiceList.SoundCloud;
import static org.schabi.newpipe.extractor.services.DefaultTests.assertNoDuplicatedItems;
@ -23,7 +22,6 @@ import org.schabi.newpipe.extractor.services.DefaultSearchExtractorTest;
import org.schabi.newpipe.extractor.utils.Utils;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.util.List;
import javax.annotation.Nullable;
@ -46,8 +44,8 @@ public class SoundcloudSearchExtractorTest {
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
// @formatter:on
@ -69,8 +67,8 @@ public class SoundcloudSearchExtractorTest {
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + urlEncode(QUERY); }
@Override public String expectedUrlContains() { return "soundcloud.com/search/tracks?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/tracks?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.STREAM; }
@ -93,8 +91,8 @@ public class SoundcloudSearchExtractorTest {
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedUrlContains() { return "soundcloud.com/search/users?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.CHANNEL; }
@ -117,8 +115,8 @@ public class SoundcloudSearchExtractorTest {
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + urlEncode(QUERY); }
@Override public String expectedUrlContains() { return "soundcloud.com/search/playlists?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/playlists?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@Override public InfoItem.InfoType expectedInfoItemType() { return InfoItem.InfoType.PLAYLIST; }
@ -139,14 +137,6 @@ public class SoundcloudSearchExtractorTest {
}
}
private static String urlEncode(String value) {
try {
return Utils.encodeUrlUtf8(value);
} catch (UnsupportedEncodingException e) {
throw new RuntimeException(e);
}
}
public static class UserVerified extends DefaultSearchExtractorTest {
private static SearchExtractor extractor;
private static final String QUERY = "David Guetta";
@ -162,8 +152,8 @@ public class SoundcloudSearchExtractorTest {
@Override public StreamingService expectedService() { return SoundCloud; }
@Override public String expectedName() { return QUERY; }
@Override public String expectedId() { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + urlEncode(QUERY); }
@Override public String expectedUrlContains() { return "soundcloud.com/search/users?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search/users?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
@ -200,8 +190,8 @@ public class SoundcloudSearchExtractorTest {
@Override public StreamingService expectedService() throws Exception { return SoundCloud; }
@Override public String expectedName() throws Exception { return QUERY; }
@Override public String expectedId() throws Exception { return QUERY; }
@Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + urlEncode(QUERY); }
@Override public String expectedUrlContains() { return "soundcloud.com/search?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedOriginalUrlContains() { return "soundcloud.com/search?q=" + Utils.encodeUrlUtf8(QUERY); }
@Override public String expectedSearchString() { return QUERY; }
@Nullable @Override public String expectedSearchSuggestion() { return null; }
}