From 38d4be2231314f009833a16c6a5b2b6f8c70c269 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:42:23 +0100 Subject: [PATCH 01/15] Update Extractor.java I add a new interface method extractSubtitleUrlfromHtml() take two string parameters, html and videoId --- .../java/com/github/kiulian/downloader/extractor/Extractor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java b/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java index 456a3d9..4aa8f0f 100644 --- a/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java +++ b/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java @@ -15,6 +15,8 @@ public interface Extractor { JSONObject extractPlayerConfigFromHtml(String html) throws YoutubeException; List extractSubtitlesLanguagesFromXml(String xml) throws YoutubeException; + + String extractSubtitleUrlfromHtml(String html, String videoId) throws YoutubeException; String extractJsUrlFromConfig(JSONObject config, String videoId) throws YoutubeException; From 3b9d55a74a7f70d25a5627bb5fd34eb884967ee3 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:44:19 +0100 Subject: [PATCH 02/15] Update ExtractorImpl.java --- .../downloader/extractor/ExtractorImpl.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java b/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java index 4effbd3..8988c15 100644 --- a/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java +++ b/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java @@ -9,6 +9,7 @@ import com.github.kiulian.downloader.downloader.Downloader; import com.github.kiulian.downloader.downloader.request.RequestWebpage; import com.github.kiulian.downloader.downloader.response.Response; +import org.apache.commons.text.StringEscapeUtils; import java.util.*; import java.util.regex.Matcher; @@ -101,6 +102,22 @@ public List extractSubtitlesLanguagesFromXml(String xml) throws YoutubeE return languages; } + @Override + public String extractSubtitleUrlfromHtml(String html, String videoId) throws YoutubeException { + String pattern = "https://www\\.youtube\\.com/api/timedtext\\?v=" + videoId + "[^\"']+"; + Pattern regex = Pattern.compile(pattern); + Matcher matcher = regex.matcher(html); + + if (!matcher.find()) { + throw new YoutubeException.BadPageException("Could not any subtitle url in the html"); + } + + String escapeUrl = matcher.group(0); + String url = StringEscapeUtils.unescapeJava(escapeUrl); + + return url; + } + @Override public String extractJsUrlFromConfig(JSONObject config, String videoId) throws YoutubeException { String js = null; From 8c7bd4a7d8447637870b27f40a618c532ad23ee3 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:50:28 +0100 Subject: [PATCH 03/15] Update ParserImpl.java I updated the parseSubtitlesInfo method and change the xmlUrl --- .../kiulian/downloader/parser/ParserImpl.java | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java b/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java index 33de12c..273d6fb 100644 --- a/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java +++ b/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java @@ -590,21 +590,26 @@ public Response> parseSubtitlesInfo(RequestSubtitlesInfo req } } - private List parseSubtitlesInfo(String videoId, YoutubeCallback> callback) throws YoutubeException { - String xmlUrl = "https://video.google.com/timedtext?hl=en&type=list&v=" + videoId; + private List parseSubtitlesInfo(String videoId, YoutubeCallback> callback) + throws YoutubeException { + String xmlUrl = "https://www.youtube.com/watch?v=" + videoId; Response response = downloader.downloadWebpage(new RequestWebpage(xmlUrl)); if (!response.ok()) { - YoutubeException e = new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", xmlUrl, response.error().getMessage())); + YoutubeException e = new YoutubeException.DownloadException( + String.format("Could not load url: %s, exception: %s", xmlUrl, response.error().getMessage())); if (callback != null) { callback.onError(e); } throw e; } - String xml = response.data(); + String html = response.data(); List languages; + + String url = ""; try { - languages = extractor.extractSubtitlesLanguagesFromXml(xml); + url = extractor.extractSubtitleUrlfromHtml(html, videoId); + languages = extractor.extractSubtitlesLanguagesFromXml(url); } catch (YoutubeException e) { if (callback != null) { callback.onError(e); @@ -614,14 +619,12 @@ private List parseSubtitlesInfo(String videoId, YoutubeCallback subtitlesInfo = new ArrayList<>(); for (String language : languages) { - String url = String.format("https://www.youtube.com/api/timedtext?lang=%s&v=%s", - language, videoId); subtitlesInfo.add(new SubtitlesInfo(url, language, false)); } return subtitlesInfo; } - + @Override public Response parseSearchResult(RequestSearchResult request) { if (request.isAsync()) { From 358af88f7c7ac4aeadd2e1ee434693c16a17da7c Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:51:03 +0100 Subject: [PATCH 04/15] Update pom.xml --- pom.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index c5eb374..9328569 100644 --- a/pom.xml +++ b/pom.xml @@ -64,7 +64,14 @@ 1.2.83 + + + org.apache.commons + commons-text + 1.7 + + - \ No newline at end of file + From 4bcf5c5e7d5c988a0baa04c0f7289fd9e489f4b1 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 20 Nov 2023 15:51:31 +0100 Subject: [PATCH 05/15] Update pom.xml --- pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/pom.xml b/pom.xml index 9328569..89895c3 100644 --- a/pom.xml +++ b/pom.xml @@ -73,5 +73,4 @@ - From a8a020d7ea19c0fb8c1701c9e5ef287d8967c6ce Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:17:52 +0100 Subject: [PATCH 06/15] Update README.md --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 380ae56..cfb09c6 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ java-youtube-downloader ============ -[![](https://jitpack.io/v/sealedtx/java-youtube-downloader.svg)](https://jitpack.io/#sealedtx/java-youtube-downloader) +[![](https://jitpack.io/v/heisfibi/java-youtube-downloader.svg)](https://jitpack.io/#heisfibi/java-youtube-downloader) Simple java parser for retrieving youtube video metadata. @@ -353,9 +353,9 @@ Include ``` ```xml - com.github.sealedtx + com.github.heisfibi java-youtube-downloader - 3.2.3 + 1.0.0 ``` @@ -371,7 +371,7 @@ allprojects { ``` ```gradle dependencies { - implementation 'com.github.sealedtx:java-youtube-downloader:3.2.3' + implementation 'com.github.heisfibi:java-youtube-downloader:1.0.0' } ``` ### Android @@ -388,4 +388,4 @@ android { jvmTarget = "1.8" } } -``` \ No newline at end of file +``` From 6fad708b5217e4b39330ec0d22877ecc5c240d51 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Tue, 21 Nov 2023 00:25:03 +0100 Subject: [PATCH 07/15] Update ParserImpl.java --- .../kiulian/downloader/parser/ParserImpl.java | 149 +++++++++++------- 1 file changed, 92 insertions(+), 57 deletions(-) diff --git a/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java b/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java index 273d6fb..413e0b9 100644 --- a/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java +++ b/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java @@ -46,7 +46,8 @@ public ParserImpl(Config config, Downloader downloader, Extractor extractor, Cip public Response parseVideo(RequestVideoInfo request) { if (request.isAsync()) { ExecutorService executorService = config.getExecutorService(); - Future result = executorService.submit(() -> parseVideo(request.getVideoId(), request.getCallback())); + Future result = executorService + .submit(() -> parseVideo(request.getVideoId(), request.getCallback())); return ResponseImpl.fromFuture(result); } try { @@ -59,7 +60,8 @@ public Response parseVideo(RequestVideoInfo request) { private VideoInfo parseVideo(String videoId, YoutubeCallback callback) throws YoutubeException { // try to spoof android - // workaround for issue https://github.com/sealedtx/java-youtube-downloader/issues/97 + // workaround for issue + // https://github.com/sealedtx/java-youtube-downloader/issues/97 VideoInfo videoInfo = parseVideoAndroid(videoId, callback); if (videoInfo == null) { videoInfo = parseVideoWeb(videoId, callback); @@ -73,8 +75,7 @@ private VideoInfo parseVideo(String videoId, YoutubeCallback callback private VideoInfo parseVideoAndroid(String videoId, YoutubeCallback callback) throws YoutubeException { String url = "https://youtubei.googleapis.com/youtubei/v1/player?key=" + ANDROID_APIKEY; - String body = - "{" + + String body = "{" + " \"videoId\": \"" + videoId + "\"," + " \"context\": {" + " \"client\": {" + @@ -129,7 +130,8 @@ private VideoInfo parseVideoWeb(String videoId, YoutubeCallback callb Response response = downloader.downloadWebpage(new RequestWebpage(htmlUrl)); if (!response.ok()) { - YoutubeException e = new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", htmlUrl, response.error().getMessage())); + YoutubeException e = new YoutubeException.DownloadException( + String.format("Could not load url: %s, exception: %s", htmlUrl, response.error().getMessage())); if (callback != null) { callback.onError(e); } @@ -169,7 +171,8 @@ private VideoInfo parseVideoWeb(String videoId, YoutubeCallback callb } throw e; } - JSONObject context = playerConfig.getJSONObject("args").getJSONObject("player_response").getJSONObject("responseContext"); + JSONObject context = playerConfig.getJSONObject("args").getJSONObject("player_response") + .getJSONObject("responseContext"); String clientVersion = extractor.extractClientVersionFromContext(context); List formats; try { @@ -202,7 +205,8 @@ private VideoDetails parseVideoDetails(String videoId, JSONObject playerResponse return new VideoDetails(videoDetails, liveHLSUrl); } - private List parseFormats(JSONObject playerResponse, String jsUrl, String clientVersion) throws YoutubeException { + private List parseFormats(JSONObject playerResponse, String jsUrl, String clientVersion) + throws YoutubeException { if (!playerResponse.containsKey("streamingData")) { throw new YoutubeException.BadPageException("streamingData not found"); } @@ -223,7 +227,8 @@ private List parseFormats(JSONObject playerResponse, String jsUrl, Strin return formats; } - private void populateFormats(List formats, JSONArray jsonFormats, String jsUrl, boolean isAdaptive, String clientVersion) throws YoutubeException.CipherException { + private void populateFormats(List formats, JSONArray jsonFormats, String jsUrl, boolean isAdaptive, + String clientVersion) throws YoutubeException.CipherException { for (int i = 0; i < jsonFormats.size(); i++) { JSONObject json = jsonFormats.getJSONObject(i); if ("FORMAT_STREAM_TYPE_OTF".equals(json.getString("type"))) @@ -251,7 +256,8 @@ private void populateFormats(List formats, JSONArray jsonFormats, String } } - private Format parseFormat(JSONObject json, String jsUrl, Itag itag, boolean isAdaptive, String clientVersion) throws YoutubeException { + private Format parseFormat(JSONObject json, String jsUrl, Itag itag, boolean isAdaptive, String clientVersion) + throws YoutubeException { if (json.containsKey("signatureCipher")) { JSONObject jsonCipher = new JSONObject(); String[] cipherData = json.getString("signatureCipher").replace("\\u0026", "&").split("&"); @@ -270,7 +276,8 @@ private Format parseFormat(JSONObject json, String jsUrl, Itag itag, boolean isA } if (urlWithSig.contains("signature") - || (!jsonCipher.containsKey("s") && (urlWithSig.contains("&sig=") || urlWithSig.contains("&lsig=")))) { + || (!jsonCipher.containsKey("s") + && (urlWithSig.contains("&sig=") || urlWithSig.contains("&lsig=")))) { // do nothing, this is pre-signed videos with signature } else if (jsUrl != null) { String s = jsonCipher.getString("s"); @@ -334,7 +341,8 @@ private List parseCaptions(JSONObject playerResponse) { public Response parsePlaylist(RequestPlaylistInfo request) { if (request.isAsync()) { ExecutorService executorService = config.getExecutorService(); - Future result = executorService.submit(() -> parsePlaylist(request.getPlaylistId(), request.getCallback())); + Future result = executorService + .submit(() -> parsePlaylist(request.getPlaylistId(), request.getCallback())); return ResponseImpl.fromFuture(result); } try { @@ -346,12 +354,14 @@ public Response parsePlaylist(RequestPlaylistInfo request) { } - private PlaylistInfo parsePlaylist(String playlistId, YoutubeCallback callback) throws YoutubeException { + private PlaylistInfo parsePlaylist(String playlistId, YoutubeCallback callback) + throws YoutubeException { String htmlUrl = "https://www.youtube.com/playlist?list=" + playlistId; Response response = downloader.downloadWebpage(new RequestWebpage(htmlUrl)); if (!response.ok()) { - YoutubeException e = new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", htmlUrl, response.error().getMessage())); + YoutubeException e = new YoutubeException.DownloadException( + String.format("Could not load url: %s, exception: %s", htmlUrl, response.error().getMessage())); if (callback != null) { callback.onError(e); } @@ -391,7 +401,8 @@ private PlaylistDetails parsePlaylistDetails(String playlistId, JSONObject initi String title = initialData.getJSONObject("metadata") .getJSONObject("playlistMetadataRenderer") .getString("title"); - JSONArray sideBarItems = initialData.getJSONObject("sidebar").getJSONObject("playlistSidebarRenderer").getJSONArray("items"); + JSONArray sideBarItems = initialData.getJSONObject("sidebar").getJSONObject("playlistSidebarRenderer") + .getJSONArray("items"); String author = null; try { // try to retrieve author, some playlists may have no author @@ -408,13 +419,15 @@ private PlaylistDetails parsePlaylistDetails(String playlistId, JSONObject initi JSONArray stats = sideBarItems.getJSONObject(0) .getJSONObject("playlistSidebarPrimaryInfoRenderer") .getJSONArray("stats"); - int videoCount = extractor.extractIntegerFromText(stats.getJSONObject(0).getJSONArray("runs").getJSONObject(0).getString("text")); + int videoCount = extractor + .extractIntegerFromText(stats.getJSONObject(0).getJSONArray("runs").getJSONObject(0).getString("text")); long viewCount = extractor.extractLongFromText(stats.getJSONObject(1).getString("simpleText")); return new PlaylistDetails(playlistId, title, author, videoCount, viewCount); } - private List parsePlaylistVideos(JSONObject initialData, int videoCount) throws YoutubeException { + private List parsePlaylistVideos(JSONObject initialData, int videoCount) + throws YoutubeException { JSONObject content; try { @@ -445,7 +458,8 @@ private List parsePlaylistVideos(JSONObject initialData, i return videos; } - private void populatePlaylist(JSONObject content, List videos, String clientVersion) throws YoutubeException { + private void populatePlaylist(JSONObject content, List videos, String clientVersion) + throws YoutubeException { JSONArray contents; if (content.containsKey("contents")) { // parse first items (up to 100) contents = content.getJSONArray("contents"); @@ -479,7 +493,8 @@ private void populatePlaylist(JSONObject content, List vid } } - private void loadPlaylistContinuation(String continuation, String ctp, List videos, String clientVersion) throws YoutubeException { + private void loadPlaylistContinuation(String continuation, String ctp, List videos, + String clientVersion) throws YoutubeException { JSONObject content; String url = "https://www.youtube.com/youtubei/v1/browse?key=" + ANDROID_APIKEY; @@ -499,7 +514,8 @@ private void loadPlaylistContinuation(String continuation, String ctp, List response = downloader.downloadWebpage(request); if (!response.ok()) { - throw new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); + throw new YoutubeException.DownloadException( + String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); } String html = response.data(); @@ -527,7 +543,8 @@ private void loadPlaylistContinuation(String continuation, String ctp, List parseChannelsUploads(RequestChannelUploads request) { if (request.isAsync()) { ExecutorService executorService = config.getExecutorService(); - Future result = executorService.submit(() -> parseChannelsUploads(request.getChannelId(), request.getCallback())); + Future result = executorService + .submit(() -> parseChannelsUploads(request.getChannelId(), request.getCallback())); return ResponseImpl.fromFuture(result); } try { @@ -538,7 +555,8 @@ public Response parseChannelsUploads(RequestChannelUploads request } } - private PlaylistInfo parseChannelsUploads(String channelId, YoutubeCallback callback) throws YoutubeException { + private PlaylistInfo parseChannelsUploads(String channelId, YoutubeCallback callback) + throws YoutubeException { String playlistId = null; if (channelId.length() == 24 && channelId.startsWith("UC")) { // channel id pattern playlistId = "UU" + channelId.substring(2); // replace "UC" with "UU" @@ -547,7 +565,8 @@ private PlaylistInfo parseChannelsUploads(String channelId, YoutubeCallback response = downloader.downloadWebpage(new RequestWebpage(channelLink)); if (!response.ok()) { - YoutubeException e = new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", channelLink, response.error().getMessage())); + YoutubeException e = new YoutubeException.DownloadException(String + .format("Could not load url: %s, exception: %s", channelLink, response.error().getMessage())); if (callback != null) { callback.onError(e); } @@ -579,7 +598,8 @@ private PlaylistInfo parseChannelsUploads(String channelId, YoutubeCallback> parseSubtitlesInfo(RequestSubtitlesInfo request) { if (request.isAsync()) { ExecutorService executorService = config.getExecutorService(); - Future> result = executorService.submit(() -> parseSubtitlesInfo(request.getVideoId(), request.getCallback())); + Future> result = executorService + .submit(() -> parseSubtitlesInfo(request.getVideoId(), request.getCallback())); return ResponseImpl.fromFuture(result); } try { @@ -590,11 +610,13 @@ public Response> parseSubtitlesInfo(RequestSubtitlesInfo req } } - private List parseSubtitlesInfo(String videoId, YoutubeCallback> callback) + private List parseSubtitlesInfo(String videoId, YoutubeCallback> callback) throws YoutubeException { String xmlUrl = "https://www.youtube.com/watch?v=" + videoId; + System.out.print("ssssssssss" + xmlUrl); Response response = downloader.downloadWebpage(new RequestWebpage(xmlUrl)); + System.out.print("ssssssssss" + response); if (!response.ok()) { YoutubeException e = new YoutubeException.DownloadException( String.format("Could not load url: %s, exception: %s", xmlUrl, response.error().getMessage())); @@ -609,27 +631,32 @@ private List parseSubtitlesInfo(String videoId, YoutubeCallback subtitlesInfo = new ArrayList<>(); for (String language : languages) { subtitlesInfo.add(new SubtitlesInfo(url, language, false)); } + System.out.print("ssssssssss" + subtitlesInfo); return subtitlesInfo; } - + @Override public Response parseSearchResult(RequestSearchResult request) { if (request.isAsync()) { ExecutorService executorService = config.getExecutorService(); - Future result = executorService.submit(() -> parseSearchResult(request.query(), request.encodeParameters(), request.getCallback())); + Future result = executorService.submit( + () -> parseSearchResult(request.query(), request.encodeParameters(), request.getCallback())); return ResponseImpl.fromFuture(result); } try { @@ -644,7 +671,8 @@ public Response parseSearchResult(RequestSearchResult request) { public Response parseSearchContinuation(RequestSearchContinuation request) { if (request.isAsync()) { ExecutorService executorService = config.getExecutorService(); - Future result = executorService.submit(() -> parseSearchContinuation(request.continuation(), request.getCallback())); + Future result = executorService + .submit(() -> parseSearchContinuation(request.continuation(), request.getCallback())); return ResponseImpl.fromFuture(result); } try { @@ -659,7 +687,8 @@ public Response parseSearchContinuation(RequestSearchContinuation public Response parseSearcheable(RequestSearchable request) { if (request.isAsync()) { ExecutorService executorService = config.getExecutorService(); - Future result = executorService.submit(() -> parseSearchable(request.searchPath(), request.getCallback())); + Future result = executorService + .submit(() -> parseSearchable(request.searchPath(), request.getCallback())); return ResponseImpl.fromFuture(result); } try { @@ -670,7 +699,8 @@ public Response parseSearcheable(RequestSearchable request) { } } - private SearchResult parseSearchResult(String query, String parameters, YoutubeCallback callback) throws YoutubeException { + private SearchResult parseSearchResult(String query, String parameters, YoutubeCallback callback) + throws YoutubeException { String searchQuery; try { searchQuery = URLEncoder.encode(query, "UTF-8"); @@ -692,7 +722,8 @@ private SearchResult parseSearchResult(String query, String parameters, YoutubeC } } - private SearchResult parseSearchable(String searchPath, YoutubeCallback callback) throws YoutubeException { + private SearchResult parseSearchable(String searchPath, YoutubeCallback callback) + throws YoutubeException { String url = "https://www.youtube.com" + searchPath; try { return parseHtmlSearchResult(url); @@ -707,11 +738,12 @@ private SearchResult parseSearchable(String searchPath, YoutubeCallback response = downloader.downloadWebpage(new RequestWebpage(url)); if (!response.ok()) { - throw new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); + throw new YoutubeException.DownloadException( + String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); } String html = response.data(); - + JSONObject initialData = extractor.extractInitialDataFromHtml(html); JSONArray rootContents; try { @@ -723,14 +755,15 @@ private SearchResult parseHtmlSearchResult(String url) throws YoutubeException { } catch (NullPointerException e) { throw new YoutubeException.BadPageException("Search result root contents not found"); } - + long estimatedCount = extractor.extractLongFromText(initialData.getString("estimatedResults")); String clientVersion = extractor.extractClientVersionFromContext(initialData.getJSONObject("responseContext")); SearchContinuation continuation = getSearchContinuation(rootContents, clientVersion); return parseSearchResult(estimatedCount, rootContents, continuation); } - private SearchResult parseSearchContinuation(SearchContinuation continuation, YoutubeCallback callback) throws YoutubeException { + private SearchResult parseSearchContinuation(SearchContinuation continuation, + YoutubeCallback callback) throws YoutubeException { String url = "https://www.youtube.com/youtubei/v1/search?key=" + ANDROID_APIKEY + "&prettyPrint=false"; JSONObject body = new JSONObject() @@ -749,7 +782,8 @@ private SearchResult parseSearchContinuation(SearchContinuation continuation, Yo Response response = downloader.downloadWebpage(request); if (!response.ok()) { - YoutubeException e = new YoutubeException.DownloadException(String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); + YoutubeException e = new YoutubeException.DownloadException( + String.format("Could not load url: %s, exception: %s", url, response.error().getMessage())); if (callback != null) { callback.onError(e); } @@ -774,7 +808,7 @@ private SearchResult parseSearchContinuation(SearchContinuation continuation, Yo } catch (Exception e) { throw new YoutubeException.BadPageException("Could not parse search continuation json"); } - + long estimatedResults = extractor.extractLongFromText(jsonResponse.getString("estimatedResults")); SearchContinuation nextContinuation = getSearchContinuation(rootContents, continuation.clientVersion()); return parseSearchResult(estimatedResults, rootContents, nextContinuation); @@ -794,7 +828,8 @@ private SearchContinuation getSearchContinuation(JSONArray rootContents, String return null; } - private SearchResult parseSearchResult(long estimatedResults, JSONArray rootContents, SearchContinuation continuation) throws BadPageException { + private SearchResult parseSearchResult(long estimatedResults, JSONArray rootContents, + SearchContinuation continuation) throws BadPageException { JSONArray contents; try { @@ -829,26 +864,26 @@ private static SearchResultElement parseSearchResultElement(JSONObject jsonItem) String rendererKey = jsonItem.keySet().iterator().next(); JSONObject jsonRenderer = jsonItem.getJSONObject(rendererKey); switch (rendererKey) { - case "videoRenderer": - return new SearchResultVideoDetails(jsonRenderer, false); - case "movieRenderer": - return new SearchResultVideoDetails(jsonRenderer, true); - case "playlistRenderer": - return new SearchResultPlaylistDetails(jsonRenderer); - case "channelRenderer": - return new SearchResultChannelDetails(jsonRenderer); - case "shelfRenderer": - return new SearchResultShelf(jsonRenderer); - case "showingResultsForRenderer": - return new QueryAutoCorrection(jsonRenderer); - case "didYouMeanRenderer": - return new QuerySuggestion(jsonRenderer); - case "horizontalCardListRenderer": - return new QueryRefinementList(jsonRenderer); - default: - System.out.println("Unknown search result element type " + rendererKey); - System.out.println(jsonItem); - return null; + case "videoRenderer": + return new SearchResultVideoDetails(jsonRenderer, false); + case "movieRenderer": + return new SearchResultVideoDetails(jsonRenderer, true); + case "playlistRenderer": + return new SearchResultPlaylistDetails(jsonRenderer); + case "channelRenderer": + return new SearchResultChannelDetails(jsonRenderer); + case "shelfRenderer": + return new SearchResultShelf(jsonRenderer); + case "showingResultsForRenderer": + return new QueryAutoCorrection(jsonRenderer); + case "didYouMeanRenderer": + return new QuerySuggestion(jsonRenderer); + case "horizontalCardListRenderer": + return new QueryRefinementList(jsonRenderer); + default: + System.out.println("Unknown search result element type " + rendererKey); + System.out.println(jsonItem); + return null; } } } From 331108f5fde25f447226f2841170c10df79a787f Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:55:22 +0100 Subject: [PATCH 08/15] Update ParserImpl.java --- .../kiulian/downloader/parser/ParserImpl.java | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java b/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java index 413e0b9..13567be 100644 --- a/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java +++ b/src/main/java/com/github/kiulian/downloader/parser/ParserImpl.java @@ -613,10 +613,8 @@ public Response> parseSubtitlesInfo(RequestSubtitlesInfo req private List parseSubtitlesInfo(String videoId, YoutubeCallback> callback) throws YoutubeException { String xmlUrl = "https://www.youtube.com/watch?v=" + videoId; - - System.out.print("ssssssssss" + xmlUrl); + Response response = downloader.downloadWebpage(new RequestWebpage(xmlUrl)); - System.out.print("ssssssssss" + response); if (!response.ok()) { YoutubeException e = new YoutubeException.DownloadException( String.format("Could not load url: %s, exception: %s", xmlUrl, response.error().getMessage())); @@ -628,26 +626,22 @@ private List parseSubtitlesInfo(String videoId, YoutubeCallback languages; - String url = ""; + String url; try { - url = extractor.extractSubtitleUrlfromHtml(html, videoId); - System.out.print("ssssssssss" + url); + url = extractor.extractSubtitleUrlFromHtml(html, videoId); languages = extractor.extractSubtitlesLanguagesFromXml(url); - System.out.print("ssssssssss" + languages); + } catch (YoutubeException e) { if (callback != null) { callback.onError(e); } throw e; } - System.out.print("ssssssssss" + url); List subtitlesInfo = new ArrayList<>(); for (String language : languages) { subtitlesInfo.add(new SubtitlesInfo(url, language, false)); } - - System.out.print("ssssssssss" + subtitlesInfo); return subtitlesInfo; } From 1eb32e9b0844483468a0c755dbc9af5ed55e4895 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:56:13 +0100 Subject: [PATCH 09/15] Update Extractor.java --- .../java/com/github/kiulian/downloader/extractor/Extractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java b/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java index 4aa8f0f..67412cf 100644 --- a/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java +++ b/src/main/java/com/github/kiulian/downloader/extractor/Extractor.java @@ -16,7 +16,7 @@ public interface Extractor { List extractSubtitlesLanguagesFromXml(String xml) throws YoutubeException; - String extractSubtitleUrlfromHtml(String html, String videoId) throws YoutubeException; + String extractSubtitleUrlFromHtml(String html, String videoId) throws YoutubeException; String extractJsUrlFromConfig(JSONObject config, String videoId) throws YoutubeException; From 1b949238d010af3fac7095404ef5bf5e3c5e9050 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:56:55 +0100 Subject: [PATCH 10/15] Update ExtractorImpl.java --- .../github/kiulian/downloader/extractor/ExtractorImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java b/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java index 8988c15..f742dc3 100644 --- a/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java +++ b/src/main/java/com/github/kiulian/downloader/extractor/ExtractorImpl.java @@ -28,7 +28,7 @@ public class ExtractorImpl implements Extractor { Pattern.compile("ytInitialData = (\\{.*?\\});") ); - private static final Pattern SUBTITLES_LANG_CODE_PATTERN = Pattern.compile("lang_code=\"(.{2,3})\""); + private static final Pattern SUBTITLES_LANG_CODE_PATTERN = Pattern.compile("lang=(.{2,3})"); private static final Pattern TEXT_NUMBER_REGEX = Pattern.compile("[0-9]+[0-9, ']*"); private static final Pattern ASSETS_JS_REGEX = Pattern.compile("\"assets\":.+?\"js\":\\s*\"([^\"]+)\""); private static final Pattern EMB_JS_REGEX = Pattern.compile("\"jsUrl\":\\s*\"([^\"]+)\""); @@ -103,7 +103,7 @@ public List extractSubtitlesLanguagesFromXml(String xml) throws YoutubeE } @Override - public String extractSubtitleUrlfromHtml(String html, String videoId) throws YoutubeException { + public String extractSubtitleUrlFromHtml(String html, String videoId) throws YoutubeException { String pattern = "https://www\\.youtube\\.com/api/timedtext\\?v=" + videoId + "[^\"']+"; Pattern regex = Pattern.compile(pattern); Matcher matcher = regex.matcher(html); From ca0e679d231b6ac00a88fb6cddb10e3e696d953b Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Wed, 22 Nov 2023 12:58:38 +0100 Subject: [PATCH 11/15] Update pom.xml --- pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 89895c3..820cd35 100644 --- a/pom.xml +++ b/pom.xml @@ -4,9 +4,9 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - com.github.ikiulian + com.github.kiulian java-youtube-downloader - 3.2.3 + 3.2.4 Java youtube video downloader Java parser for retrieving youtube video meta info From 0c5838ce70efc5449049d45f77c5775a595ce621 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Wed, 22 Nov 2023 14:29:13 +0100 Subject: [PATCH 12/15] Update README.md --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index cfb09c6..59ec170 100644 --- a/README.md +++ b/README.md @@ -355,7 +355,7 @@ Include com.github.heisfibi java-youtube-downloader - 1.0.0 + 1.0.1 ``` @@ -371,7 +371,7 @@ allprojects { ``` ```gradle dependencies { - implementation 'com.github.heisfibi:java-youtube-downloader:1.0.0' + implementation 'com.github.heisfibi:java-youtube-downloader:1.0.1' } ``` ### Android From bbc106c3d1286c2d6f33487247a456db4fd918e7 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 27 Nov 2023 00:11:00 +0100 Subject: [PATCH 13/15] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 59ec170..18d083f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ java-youtube-downloader ============ -[![](https://jitpack.io/v/heisfibi/java-youtube-downloader.svg)](https://jitpack.io/#heisfibi/java-youtube-downloader) +[![](https://jitpack.io/v/sealedtx/java-youtube-downloader.svg)](https://jitpack.io/#sealedtx/java-youtube-downloader) Simple java parser for retrieving youtube video metadata. From e87387c7bbbcc4f6d6c8a444d32d72b6095b5639 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 27 Nov 2023 00:14:51 +0100 Subject: [PATCH 14/15] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 18d083f..22d1a4b 100644 --- a/README.md +++ b/README.md @@ -353,9 +353,9 @@ Include ``` ```xml - com.github.heisfibi + com.github.sealedtx java-youtube-downloader - 1.0.1 + 3.2.3 ``` @@ -371,7 +371,7 @@ allprojects { ``` ```gradle dependencies { - implementation 'com.github.heisfibi:java-youtube-downloader:1.0.1' + implementation 'com.github.sealedtx:java-youtube-downloader:3.2.3' } ``` ### Android From 974eb94448bb353a214ba840f9bec6c5ddd5ed26 Mon Sep 17 00:00:00 2001 From: Richard Iyoke <66504142+heisfibi@users.noreply.github.com> Date: Mon, 27 Nov 2023 00:16:55 +0100 Subject: [PATCH 15/15] Update README.md