From 64c64c821d8762a1b4d5a489d479228133b95aac Mon Sep 17 00:00:00 2001 From: trizen Date: Sat, 31 Oct 2020 21:46:13 +0200 Subject: [PATCH] Implemented support for listing a playlist of videos by playlistID. --- README.md | 2 +- lib/WWW/PipeViewer/Channels.pm | 8 +- lib/WWW/PipeViewer/InitialData.pm | 236 +++++++++++++++++++++------- lib/WWW/PipeViewer/PlaylistItems.pm | 12 +- lib/WWW/PipeViewer/Playlists.pm | 2 +- lib/WWW/PipeViewer/Search.pm | 6 +- lib/WWW/PipeViewer/Utils.pm | 64 +++----- 7 files changed, 223 insertions(+), 107 deletions(-) diff --git a/README.md b/README.md index bc498e3..c4237d2 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ A lightweight application (fork of [straw-viewer](https://github.com/trizen/straw-viewer)) for searching and playing videos from YouTube, using the [API](https://github.com/iv-org/invidious/wiki/API) of [invidio.us](https://invidio.us/). -The goal of this fork is to parse the YouTube website directly and rely on the invidious instances only as fallback methods. +The goal of this fork is to parse the YouTube website directly and rely on the invidious instances only as a fallback method. ### pipe-viewer diff --git a/lib/WWW/PipeViewer/Channels.pm b/lib/WWW/PipeViewer/Channels.pm index 702215c..83deec1 100644 --- a/lib/WWW/PipeViewer/Channels.pm +++ b/lib/WWW/PipeViewer/Channels.pm @@ -28,11 +28,11 @@ sub videos_from_channel_id { my $url = $self->_make_feed_url("channels/$channel_id/videos"); - if (my @results = $self->_channel_uploads($channel_id)) { + if (my @results = $self->yt_channel_uploads($channel_id)) { return { - url => $url, - results => \@results, - }; + url => $url, + results => \@results, + }; } return $self->_get_results($url); diff --git a/lib/WWW/PipeViewer/InitialData.pm b/lib/WWW/PipeViewer/InitialData.pm index cd9ab56..3e2287f 100644 --- a/lib/WWW/PipeViewer/InitialData.pm +++ b/lib/WWW/PipeViewer/InitialData.pm @@ -13,6 +13,9 @@ WWW::PipeViewer::InitialData - Extract initial data. use WWW::PipeViewer; my $obj = WWW::PipeViewer->new(%opts); + my $results = $obj->yt_search(q => $keywords); + my $playlists = $obj->yt_channel_playlists($channel_ID); + =head1 SUBROUTINES/METHODS =cut @@ -120,6 +123,86 @@ sub _extract_youtube_mix { return \%mix; } +sub _extract_author_name { + my ($info) = @_; + eval { $info->{longBylineText}{runs}[0]{text} } // eval { $info->{shortBylineText}{runs}[0]{text} }; +} + +sub _extract_video_id { + my ($info) = @_; + eval { $info->{videoId} } || eval { $info->{navigationEndpoint}{watchEndpoint}{videoId} } || undef; +} + +sub _extract_length_seconds { + my ($info) = @_; + eval { $info->{lengthSeconds} } + || _time_to_seconds(eval { $info->{thumbnailOverlays}[0]{thumbnailOverlayTimeStatusRenderer}{text}{runs}[0]{text} } // 0) + || _time_to_seconds(eval { $info->{lengthText}{runs}[0]{text} // 0 }); +} + +sub _extract_published_text { + my ($info) = @_; + eval { $info->{publishedTimeText}{runs}[0]{text} }; +} + +sub _extract_channel_id { + my ($info) = @_; + eval { $info->{channelId} } // eval { $info->{shortBylineText}{runs}[0]{navigationEndpoint}{browseEndpoint}{browseId} }; +} + +sub _extract_view_count_text { + my ($info) = @_; + eval { $info->{shortViewCountText}{runs}[0]{text} }; +} + +sub _extract_video_thumbnails { + my ($info) = @_; + eval { + [ + map { + my %thumb = %$_; + $thumb{quality} = _thumbnail_quality($thumb{width}, $thumb{height}); + $thumb{url} = _fix_url_protocol($thumb{url}); + \%thumb; + } @{$info->{thumbnail}{thumbnails}} + ] + }; +} + +sub _extract_title { + my ($info) = @_; + eval { $info->{title}{runs}[0]{text} } // eval { $info->{title}{accessibility}{accessibilityData}{label} }; +} + +sub _extract_description { + my ($info) = @_; + + # FIXME: this is not the video description + eval { $info->{title}{accessibility}{accessibilityData}{label} }; +} + +sub _extract_view_count { + my ($info) = @_; + _human_number_to_int(eval { $info->{viewCountText}{runs}[0]{text} } // 0); +} + +sub _extract_video_count { + my ($info) = @_; + _human_number_to_int(eval { $info->{videoCountShortText}{runs}[0]{text} } + // eval { $info->{videoCountText}{runs}[0]{text} } // 0); +} + +sub _extract_playlist_id { + my ($info) = @_; + eval { $info->{playlistId} }; +} + +sub _extract_playlist_thumbnail { + my ($info) = @_; + eval { _fix_url_protocol($info->{thumbnailRenderer}{playlistVideoThumbnailRenderer}{thumbnail}{thumbnails}[0]{url}) } + // eval { _fix_url_protocol($info->{thumbnail}{thumbnails}[0]{url}) }; +} + sub _extract_itemSection_entry { my ($self, $data, %args) = @_; @@ -129,64 +212,44 @@ sub _extract_itemSection_entry { } # Video - if (exists $data->{compactVideoRenderer}) { + if (exists($data->{compactVideoRenderer}) or exists($data->{playlistVideoRenderer})) { my %video; - my $info = $data->{compactVideoRenderer}; + my $info = eval { $data->{compactVideoRenderer} } // eval { $data->{playlistVideoRenderer} }; $video{type} = 'video'; - $video{title} = - eval { $info->{title}{runs}[0]{text} } // eval { $info->{title}{accessibility}{accessibilityData}{label} } // return; + # Deleted video + if (defined(eval { $info->{isPlayable} }) and not $info->{isPlayable}) { + return; + } - $video{videoId} = eval { $info->{navigationEndpoint}{watchEndpoint}{videoId} } // $info->{videoId} // return; - $video{author} = eval { $info->{longBylineText}{runs}[0]{text} } // eval { $info->{shortBylineText}{runs}[0]{text} }; - $video{authorId} = eval { $info->{channelId} }; - $video{publishedText} = eval { $info->{publishedTimeText}{runs}[0]{text} }; - $video{viewCountText} = eval { $info->{shortViewCountText}{runs}[0]{text} }; - - $video{videoThumbnails} = eval { - [ - map { - my %thumb = %$_; - $thumb{quality} = _thumbnail_quality($thumb{width}, $thumb{height}); - $thumb{url} = _fix_url_protocol($thumb{url}); - \%thumb; - } @{$info->{thumbnail}{thumbnails}} - ] - }; - - # FIXME: this is not the video description - $video{description} = eval { $info->{title}{accessibility}{accessibilityData}{label} }; - $video{lengthSeconds} = _time_to_seconds( - eval { - $info->{thumbnailOverlays}[0]{thumbnailOverlayTimeStatusRenderer}{text}{runs}[0]{text}; - } // 0 - ); - $video{title} = eval { $info->{title}{runs}[0]{text} }; - $video{viewCount} = _human_number_to_int(eval { $info->{viewCountText}{runs}[0]{text} } // 0); + $video{videoId} = _extract_video_id($info) // return; + $video{title} = _extract_title($info) // return; + $video{lengthSeconds} = _extract_length_seconds($info) || return; + $video{author} = _extract_author_name($info); + $video{authorId} = _extract_channel_id($info); + $video{publishedText} = _extract_published_text($info); + $video{viewCountText} = _extract_view_count_text($info); + $video{videoThumbnails} = _extract_video_thumbnails($info); + $video{description} = _extract_description($info); + $video{viewCount} = _extract_view_count($info); return \%video; } # Playlist - if (exists $data->{compactPlaylistRenderer}) { + if ($args{type} ne 'video' and exists $data->{compactPlaylistRenderer}) { my %playlist; - my $info = $data->{compactPlaylistRenderer}; + my $info = eval { $data->{compactPlaylistRenderer} }; $playlist{type} = 'playlist'; - $playlist{title} = - eval { $info->{title}{runs}[0]{text} } // eval { $info->{title}{accessibility}{accessibilityData}{label} } // return; - - $playlist{playlistId} = $info->{playlistId}; - $playlist{videoCount} = _human_number_to_int(eval { $info->{videoCountShortText}{runs}[0]{text} } - // eval { $info->{videoCountText}{runs}[0]{text} } // 0); - - $playlist{playlistThumbnail} = - eval { _fix_url_protocol($info->{thumbnailRenderer}{playlistVideoThumbnailRenderer}{thumbnail}{thumbnails}[0]{url}) } - // eval { _fix_url_protocol($info->{thumbnail}{thumbnails}[0]{url}) }; + $playlist{title} = _extract_title($info) // return; + $playlist{playlistId} = _extract_playlist_id($info) // return; + $playlist{videoCount} = _extract_video_count($info); + $playlist{playlistThumbnail} = _extract_playlist_thumbnail($info); return \%playlist; } @@ -205,7 +268,7 @@ sub _parse_itemSection { my $item = $self->_extract_itemSection_entry($entry, %args); - if (defined($item)) { + if (defined($item) and ref($item) eq 'HASH') { push @results, $item; } } @@ -228,6 +291,13 @@ sub _extract_sectionList_results { $self->_parse_itemSection({contents => $entry->{shelfRenderer}{content}{verticalListRenderer}{items}}, %args); } + # Playlist videos + if (eval { ref($entry->{itemSectionRenderer}{contents}[0]{playlistVideoListRenderer}{contents}) eq 'ARRAY' }) { + push @results, + $self->_parse_itemSection($entry->{itemSectionRenderer}{contents}[0]{playlistVideoListRenderer}, %args); + next; + } + # YouTube Mix if ($args{type} eq 'all' and exists $entry->{universalWatchCardRenderer}) { @@ -238,7 +308,7 @@ sub _extract_sectionList_results { } } - # Search results + # Video results if (exists $entry->{itemSectionRenderer}) { push @results, $self->_parse_itemSection($entry->{itemSectionRenderer}, %args); } @@ -296,6 +366,19 @@ sub _extract_channel_playlists { return @results; } +sub _extract_playlist_videos { + my ($self, $data, %args) = @_; + + my @results = $self->_extract_sectionList_results( + eval { + $data->{contents}{singleColumnBrowseResultsRenderer}{tabs}[0]{tabRenderer}{content}{sectionListRenderer}; + }, + %args + ); + $self->_add_author_to_results($data, \@results, %args); + return @results; +} + sub _get_initial_data { my ($self, $url) = @_; @@ -310,17 +393,6 @@ sub _get_initial_data { return; } -sub _youtube_search { - my ($self, %args) = @_; - - my $url = $self->get_m_youtube_url . "/results?search_query=$args{q}"; - - # TODO: add support for various search parameters - - my $hash = $self->_get_initial_data($url) // return; - $self->_extract_sectionList_results(eval { $hash->{contents}{sectionListRenderer} }, %args); -} - sub _channel_data { my ($self, $channel, %args) = @_; @@ -338,18 +410,68 @@ sub _channel_data { $self->_get_initial_data($url); } -sub _channel_uploads { +=head2 yt_search(q => $keyword, %args) + +Search for videos given a keyword (uri-escaped). + +=cut + +sub yt_search { + my ($self, %args) = @_; + + my $url = $self->get_m_youtube_url . "/results?search_query=$args{q}"; + + # TODO: add support for various search parameters + + my $hash = $self->_get_initial_data($url) // return; + $self->_extract_sectionList_results(eval { $hash->{contents}{sectionListRenderer} }, %args); +} + +=head2 yt_channel_uploads($channel, %args) + +Latest uploads for a given channel ID or username. + +=cut + +sub yt_channel_uploads { my ($self, $channel, %args) = @_; my $hash = $self->_channel_data($channel, type => 'videos') // return; $self->_extract_channel_uploads($hash, %args, type => 'video'); } -sub _channel_playlists { +=head2 yt_channel_playlists($channel, %args) + +Playlists for a given channel ID or username. + +=cut + +sub yt_channel_playlists { my ($self, $channel, %args) = @_; my $hash = $self->_channel_data($channel, type => 'playlists') // return; $self->_extract_channel_playlists($hash, %args, type => 'playlist'); } +=head2 yt_playlist_videos($playlist_id, %args) + +Videos from a given playlist ID. + +=cut + +sub yt_playlist_videos { + my ($self, $playlist_id, %args) = @_; + + my $url = $self->get_m_youtube_url . "/playlist?list=$playlist_id"; + my $hash = $self->_get_initial_data($url) // return; + + $self->_extract_sectionList_results( + eval { + $hash->{contents}{singleColumnBrowseResultsRenderer}{tabs}[0]{tabRenderer}{content}{sectionListRenderer}; + }, + %args, + type => 'video' + ); +} + =head1 AUTHOR Trizen, C<< >> diff --git a/lib/WWW/PipeViewer/PlaylistItems.pm b/lib/WWW/PipeViewer/PlaylistItems.pm index 5a70e9f..0d33735 100644 --- a/lib/WWW/PipeViewer/PlaylistItems.pm +++ b/lib/WWW/PipeViewer/PlaylistItems.pm @@ -80,7 +80,17 @@ Get videos from a specific playlistID. sub videos_from_playlist_id { my ($self, $id) = @_; - $self->_get_results($self->_make_feed_url("playlists/$id")); + + my $url = $self->_make_feed_url("playlists/$id"); + + if (my @results = $self->yt_playlist_videos($id)) { + return { + url => $url, + results => \@results, + }; + } + + $self->_get_results($url); } =head2 favorites($channel_id) diff --git a/lib/WWW/PipeViewer/Playlists.pm b/lib/WWW/PipeViewer/Playlists.pm index debba81..27daf31 100644 --- a/lib/WWW/PipeViewer/Playlists.pm +++ b/lib/WWW/PipeViewer/Playlists.pm @@ -63,7 +63,7 @@ sub playlists { my $url = $self->_make_feed_url("channels/playlists/$channel_id"); - if (my @results = $self->_channel_playlists($channel_id)) { + if (my @results = $self->yt_channel_playlists($channel_id)) { return scalar { url => $url, diff --git a/lib/WWW/PipeViewer/Search.pm b/lib/WWW/PipeViewer/Search.pm index 4bd546d..ee0908b 100644 --- a/lib/WWW/PipeViewer/Search.pm +++ b/lib/WWW/PipeViewer/Search.pm @@ -97,12 +97,12 @@ sub search_for { if ($type eq 'video' and $url =~ /\?q=[^&]+&type=video\z/) { - if (my @results = $self->_youtube_search(q => $keywords, type => $type, %$args)) { + if (my @results = $self->yt_search(q => $keywords, type => $type, %$args)) { return { url => $url, results => \@results, - }; - } + }; + } } return $self->_get_results($url); diff --git a/lib/WWW/PipeViewer/Utils.pm b/lib/WWW/PipeViewer/Utils.pm index dc6aa1b..940e7e7 100644 --- a/lib/WWW/PipeViewer/Utils.pm +++ b/lib/WWW/PipeViewer/Utils.pm @@ -225,7 +225,7 @@ sub has_entries { if (ref($result->{results}) eq 'HASH') { - foreach my $type(qw(comments videos playlists)) { + foreach my $type (qw(comments videos playlists)) { if (exists $result->{results}{$type}) { return scalar @{$result->{results}{$type}} > 0; } @@ -246,8 +246,8 @@ sub has_entries { return 0; } - return 1; # maybe? - #ref($result) eq 'HASH' and ($result->{results}{pageInfo}{totalResults} > 0); + return 1; # maybe? + #ref($result) eq 'HASH' and ($result->{results}{pageInfo}{totalResults} > 0); } =head2 normalize_video_title($title, $fat32safe) @@ -384,8 +384,8 @@ sub format_text { $text =~ s/$escapes_re/$special_escapes{$1}/g; $escape - ? $text =~ s/$tokens_re/\Q${\$special_tokens{$1}()}\E/gr - : $text =~ s/$tokens_re/${\$special_tokens{$1}()}/gr; + ? $text =~ s<$tokens_re><\Q${\($special_tokens{$1}() // '')}\E>gr + : $text =~ s<$tokens_re><${\($special_tokens{$1}() // '')}>gr; } =head2 set_thousands($num) @@ -421,22 +421,6 @@ Get videoID. sub get_video_id { my ($self, $info) = @_; $info->{videoId}; - - #~ ref($info->{id}) eq 'HASH' ? $info->{id}{videoId} - #~ : exists($info->{snippet}{resourceId}{videoId}) ? $info->{snippet}{resourceId}{videoId} - #~ : exists($info->{contentDetails}{videoId}) ? $info->{contentDetails}{videoId} - #~ : exists($info->{contentDetails}{playlistItem}{resourceId}{videoId}) - #~ ? $info->{contentDetails}{playlistItem}{resourceId}{videoId} - #~ : exists($info->{contentDetails}{upload}{videoId}) ? $info->{contentDetails}{upload}{videoId} - #~ : do { - #~ my $id = $info->{id} // return undef; - - #~ if (length($id) != 11) { - #~ return undef; - #~ } - - #~ $id; - #~ }; } sub get_playlist_id { @@ -543,8 +527,8 @@ sub get_thumbnail_url { ref($info->{videoThumbnails}) eq 'ARRAY' or return ''; - my @thumbs = @{$info->{videoThumbnails}}; - my @wanted = grep{$_->{quality} eq $type} @thumbs; + my @thumbs = @{$info->{videoThumbnails}}; + my @wanted = grep { $_->{quality} eq $type } @thumbs; my $url; @@ -564,6 +548,7 @@ sub get_thumbnail_url { sub get_channel_title { my ($self, $info) = @_; + #$info->{snippet}{channelTitle} || $self->get_channel_id($info); $info->{author}; } @@ -585,18 +570,21 @@ sub get_comment_content { sub get_id { my ($self, $info) = @_; + #$info->{id}; $info->{videoId}; } sub get_channel_id { my ($self, $info) = @_; + #$info->{snippet}{resourceId}{channelId} // $info->{snippet}{channelId}; $info->{authorId}; } sub get_category_id { my ($self, $info) = @_; + #$info->{snippet}{resourceId}{categoryId} // $info->{snippet}{categoryId}; #"unknown"; $info->{genre} // 'Unknown'; @@ -630,6 +618,7 @@ sub get_category_name { sub get_publication_date { my ($self, $info) = @_; + #$self->format_date($info->{snippet}{publishedAt}); #$self->format_date require Time::Piece; @@ -639,7 +628,7 @@ sub get_publication_date { sub get_publication_age { my ($self, $info) = @_; - ($info->{publishedText} // '') =~ s/\sago\z//r;; + ($info->{publishedText} // '') =~ s/\sago\z//r; } sub get_publication_age_approx { @@ -672,6 +661,7 @@ sub get_publication_age_approx { sub get_duration { my ($self, $info) = @_; + #$self->format_duration($info->{contentDetails}{duration}); #$self->format_duration($info->{lengthSeconds}); $info->{lengthSeconds}; @@ -691,6 +681,7 @@ sub get_time { sub get_definition { my ($self, $info) = @_; + #uc($info->{contentDetails}{definition} // '-'); #...; "unknown"; @@ -698,6 +689,7 @@ sub get_definition { sub get_dimension { my ($self, $info) = @_; + #uc($info->{contentDetails}{dimension}); #...; "unknown"; @@ -705,6 +697,7 @@ sub get_dimension { sub get_caption { my ($self, $info) = @_; + #$info->{contentDetails}{caption}; #...; "unknown"; @@ -762,14 +755,14 @@ sub get_dislikes { sub get_comments { my ($self, $info) = @_; + #$info->{statistics}{commentCount}; 1; } { no strict 'refs'; - foreach my $pair ( - [playlist => {'playlist' => 1}], + foreach my $pair ([playlist => {'playlist' => 1}], [channel => {'channel' => 1}], [video => {'video' => 1, 'playlistItem' => 1}], [subscription => {'subscription' => 1}], @@ -780,23 +773,14 @@ sub get_comments { my ($self, $item) = @_; if ($pair->[0] eq 'video') { - return 1 if exists $item->{videoId}; + return 1 if defined $item->{videoId}; + } + + if ($pair->[0] eq 'playlist') { + return 1 if defined $item->{playlistId}; } exists $pair->[1]{$item->{type} // ''}; - - #~ if (ref($item->{id}) eq 'HASH') { - #~ if (exists $pair->[1]{$item->{id}{kind}}) { - #~ return 1; - #~ } - #~ } - #~ elsif (exists $item->{kind}) { - #~ if (exists $pair->[1]{$item->{kind}}) { - #~ return 1; - #~ } - #~ } - - #~ return; }; }