8 Commits

Author SHA1 Message Date
stabbedbybrick
8a8be61a3b added search option 2023-10-15 18:07:24 +02:00
stabbedbybrick
55fa84c518 fixed broken content 2023-10-15 18:07:14 +02:00
stabbedbybrick
4cfd580caf update 2023-10-15 18:06:08 +02:00
stabbedbybrick
91676b4892 improved dynamic imports 2023-10-15 18:05:58 +02:00
stabbedbybrick
bca710fbf9 update 2023-10-15 18:05:17 +02:00
stabbedbybrick
6e505072c5 added search option 2023-10-15 18:05:05 +02:00
stabbedbybrick
b84195510e update 2023-10-15 18:04:37 +02:00
stabbedbybrick
3976a63f01 update 2023-10-15 18:04:23 +02:00
9 changed files with 657 additions and 92 deletions

View File

@@ -8,6 +8,7 @@ Download videos from free streaming services
- [x] Automatic PSSH, manifest, and key retreival
- [x] Local and remote CDM options
- [x] Config file with settings for download path, file format, subtitle options etc.
- [x] Search option
#### Supported services:
(Premium content on any service is not supported)

View File

@@ -22,7 +22,7 @@ subtitles:
clean: "true" # Clean and convert subtitles. If "false", subtitles remain untouched
# Customize filename output for series and movies
# Default: Title.S01E01.EpisodeName.1080p.SERVICE.WEB-DL.AUDIO.CODEC
# Default file names follow the current P2P standard: Title.S01E01.EpisodeName.1080p.SERVICE.WEB-DL.AUDIO.CODEC
# NOTE: {year} is dependent on if the service API has it configured, which is hit or miss
# Manually adding the year for each series is recommended if you need it included
filename:

View File

@@ -9,10 +9,12 @@ from helpers import __version__
from helpers.documentation import main_help
from helpers.services import get_service
from helpers.utilities import info
from helpers.search import search_engine
@click.command(help=main_help)
@click.argument("url", type=str, required=True)
@click.option("--search", nargs=2, type=str, help="Search service(s) for titles")
@click.argument("url", type=str, required=False)
@click.option("-q", "--quality", type=str, help="Specify resolution")
@click.option("-a", "--all-audio", is_flag=True, help="Include all audio tracks")
@click.option("-e", "--episode", type=str, help="Download episode(s)")
@@ -22,18 +24,22 @@ from helpers.utilities import info
@click.option("-t", "--titles", is_flag=True, default=False, help="List all titles")
@click.option("-i", "--info", is_flag=True, default=False, help="Print title info")
@click.option("-r", "--remote", is_flag=True, default=False, help="Use remote CDM")
def main(**kwargs) -> None:
def main(search=None, **kwargs) -> None:
click.echo("")
info(f"Freevine {__version__}\n")
with open("config.yaml", "r") as f:
config = yaml.safe_load(f)
if search:
alias, keywords = search
search_engine(alias, keywords)
else:
with open("config.yaml", "r") as f:
config = yaml.safe_load(f)
with open(Path("services") / "services.yaml", "r") as f:
srvc = yaml.safe_load(f)
with open(Path("services") / "services.yaml", "r") as f:
srvc = yaml.safe_load(f)
Service = get_service(kwargs.get("url"))
Service(config, srvc, **kwargs)
Service = get_service(kwargs.get("url"))
Service(config, srvc, **kwargs)
shutil.rmtree("tmp") if Path("tmp").exists() else None

View File

@@ -1 +1 @@
__version__ = "v0.5.7-beta (20231011)"
__version__ = "v0.5.8-beta (20231015)"

137
helpers/dict.py Normal file
View File

@@ -0,0 +1,137 @@
def _search(keywords: str):
return [
{
"name": "BBC iPlayer",
"alias": ["BBC", "IPLAYER", "BBCIPLAYER"],
"url": "https://search.api.bbci.co.uk/formula/iplayer-ibl-root",
"params": {
"q": f"{keywords}",
"apikey": "HJ34sajBaTjACnUJtGZ2Gvsy0QeqJ5UK",
},
"method": "GET",
},
{
"name": "ALL4",
"alias": ["ALL4", "CHANNEL4", "C4", "CH4"],
"url": "https://all4nav.channel4.com/v1/api/search",
"params": {
"expand": "default",
"q": f"{keywords}",
"limit": "100",
"offset": "0",
},
"method": "GET",
},
{
"name": "My5",
"alias": ["MY5", "CHANNEL5", "C5", "CH5"],
"url": "https://corona.channel5.com/shows/search.json",
"params": {
"platform": "my5desktop",
"friendly": "1",
"query": f"{keywords}",
},
"method": "GET",
},
{
"name": "CRACKLE",
"alias": ["CRACKLE", "CRKL"],
"url": f"https://prod-api.crackle.com/contentdiscovery/search/{keywords}",
"header": {"x-crackle-platform": "5FE67CCA-069A-42C6-A20F-4B47A8054D46"},
"params": {
"useFuzzyMatching": "false",
"enforcemediaRights": "true",
"pageNumber": "1",
"pageSize": "20",
"contentType": "Channels",
"searchFields": "Title,Cast",
},
"method": "GET",
},
{
"name": "CTV",
"alias": ["CTV"],
"url": "https://www.ctv.ca/space-graphql/apq/graphql",
"payload": {
"operationName": "searchMedia",
"variables": {"title": f"{keywords}"},
"query": """
query searchMedia($title: String!) {searchMedia(titleMatches: $title) {
... on Medias {page {items {title\npath}}}}}, """,
},
"method": "POST",
},
{
"name": "ITV",
"alias": ["ITV", "ITVX"],
"url": "https://textsearch.prd.oasvc.itv.com/search",
"params": {
"broadcaster": "itv",
"featureSet": "clearkey,outband-webvtt,hls,aes,playready,widevine,fairplay,bbts,progressive,hd,rtmpe",
"onlyFree": "false",
"platform": "dotcom",
"query": f"{keywords}",
},
"method": "GET",
},
{
"name": "PlutoTV",
"alias": ["PLUTOTV", "PLUTO"],
"url": "https://service-media-search.clusters.pluto.tv/v1/search",
"params": {
"q": f"{keywords}",
"limit": "100",
},
"method": "GET",
},
{
"name": "The Roku Channel",
"alias": ["ROKU", "ROKUCHANNEL", "THEROKUCHANNEL"],
"url": "https://therokuchannel.roku.com/api/v1/search",
"token": "https://therokuchannel.roku.com/api/v1/csrf",
"payload": {
"query": f"{keywords}",
},
"method": "POST",
},
{
"name": "STV Player",
"alias": ["STV", "STVPLAYER"],
"url": "https://search-api.swiftype.com/api/v1/public/engines/suggest.json",
"params": None,
"method": "POST",
"payload": {
"engine_key": "S1jgssBHdk8ZtMWngK_y",
"per_page": 10,
"page": 1,
"fetch_fields": {"page": ["title", "body", "resultDescriptionTx", "url"]},
"search_fields": {"page": ["title^3", "body"]},
"q": f"{keywords}",
"spelling": "strict",
},
},
{
"name": "TubiTV",
"alias": ["TUBI"],
"url": f"https://tubitv.com/oz/search/{keywords}",
"collect": {
"connect.sid": "s%3Al5xcbiTUygyjM1olYs6zqLwQuqEtdTuU.8Z%2B0IcWqpmn4De9thyYAkjJ7rFe9FIj%2FmHOQxtXnbxs"
},
"params": {
"isKidsMode": "false",
"useLinearHeader": "true",
"isMobile": "false",
},
"method": "GET",
},
{
"name": "UKTV Play",
"alias": ["UKTV", "UKTVP", "UKTVPLAY"],
"url": "https://vschedules.uktv.co.uk/vod/search/",
"params": {
"q": f"{keywords}",
},
"method": "GET",
},
]

View File

@@ -8,75 +8,109 @@ main_help = f"""
\b
Requirements:
Python 3.9+
Valid L3 CDM(blob and key)
Valid L3 CDM(place in pywidevine/L3/cdm/devices/android_generic)
N_m3u8DL-RE
ffmpeg
mkvmerge
mp4decrypt
\b
Python packages installation:
pip install -r requirements.txt
\b
Settings:
Open config.yaml in your favorite text editor to change settings like
download path, folder structure, filenames, subtitle options etc.
Open config.yaml in your favorite text editor to configure
download paths, folder structure, filenames, subtitle options etc.
\b
Instructions:
Place blob and key file in pywidevine/L3/cdm/devices/android_generic to use local CDM
Use --remote option if you don't have a CDM (ALL4 not supported)
This program has got two methods of downloading:
\b
Use freevine.py followed by options and URL to content
Service is found automatically and is not needed in the command
See examples at the bottom for usage
Method 1: (singles and batch)
Provide the series main URL and request what to download from it:
\b
Always use main URL of series for batch downloads, not episode URLs
Use the "S01E01" format (Season 1, Episode 1) to request episodes
Use --episode S01E01-S01E10 to request a range of episodes (from the same season)
Use --episode S01E01,S03E07,S10E12 (no spaces!) to request a mix of episodes
Use --season S01,S03,S10 (no spaces!) to request a mix of seasons
python freevine.py --episode S01E01 URL
python freevine.py --season S01 URL
python freevine.py --episode S01E01-S01E10 URL
python freevine.py --episode S01E01,S03E07,S10E12 URL
python freevine.py --season S01,S03,S10 URL
python freevine.py --complete URL
\b
--titles to list all available episodes from a series
--remote to get decryption keys remotely (default: local CDM)
--info to print description and all available quality profiles from a title
--quality to specify video quality (default: Best)
--all-audio to include all audio tracks (default: Best)
NOTES:
Always use main URL of series for this method, not episode URLs
Use the "S01E01" format (Season 1, Episode 1) to request episodes
Use --episode S01E01-S01E10 to request a range of episodes (from the same season)
Use --episode S01E01,S03E07,S10E12 (no spaces!) to request a mix of episodes
Use --season S01,S03,S10 (no spaces!) to request a mix of seasons
\b
Information:
Method 2: (singles)
Provide URL to episode or movie to download it directly:
\b
python freevine.py EPISODE_URL
python freevine.py --movie MOVIE_URL
\b
NOTES:
Grabbing the URLs straight from the frontpage often comes with extra
garbage attached. It's recommended to get the URL from title page
\b
Options:
List all available episodes from a series:
python freevine.py --titles URL
Print available quality streams and info about a single title:
python freevine.py --info --episode URL
python freevine.py --info --movie URL
Request video quality to be downloaded: (default: best)
python freevine.py --quality 1080p --episode/--season URL
python freevine.py --quality 1080p --movie URL
Request all audio tracks to be downloaded: (default: best)
python freevine.py --all-audio --episode/--season URL
python freevine.py --all-audio --movie URL
Use remote CDM (ALL4 not supported):
python freevine.py --remote --episode/--season URL
\b
NOTES:
The order of the options isn't super strict, but it's recommended to follow the examples above
Combinations are possible as far as common sense goes
If you request a quality that's not available, the closest match is downloaded instead
\b
Searching (beta):
You can use the search option to search for titles in the command line:
\b
python freevine.py --search all4 "QUERY"
python freevine.py --search all4,ctv,itv "QUERY"
\b
NOTES:
You can search one or multiple services at the same time
The results should produce usable URL to series or movie
Some services have geo block even for searching
\b
Service information:
(Premium content on any service is not supported)
ROKU: 1080p, DD5.1
CTV: 1080p, DD5.1
ALL4: 1080p, AAC2.0
MY5: 1080p, AAC2.0
iPLAYER: 1080p, AAC2.0
UKTVPLAY: 1080p, AAC2.0
STV: 1080p, AAC2.0
CRACKLE: 1080p, AAC2.0
ITV: 720p, AAC2.0
TUBI: 720p, AAC2.0
PLUTO: 720p, AAC2.0
\b
Default file names follow the current P2P standard:
"Title.S01E01.Name.1080p.SERVICE.WEB-DL.AUDIO.CODEC"
NOTE: Soaps and certain programmes without clear labels might display
odd names and numbers. It's a known bug.
ROKU: 1080p, DD5.1
CTV: 1080p, DD5.1
ALL4: 1080p, AAC2.0 *
MY5: 1080p, AAC2.0
iPLAYER: 1080p, AAC2.0
UKTVPLAY: 1080p, AAC2.0
STV: 1080p, AAC2.0
CRACKLE: 1080p, AAC2.0
ITV: 720p, AAC2.0
TUBI: 720p, AAC2.0
PLUTO: 720p, AAC2.0
\b
If you request a quality that's not available,
the closest match is downloaded instead
*ALL4 offer different quality streams on different API endpoints
You can switch between them in /services/services.yaml by using "android" or "web" as client
\b
Final notes:
\b
This program is just a hobby project inbetween real-life responsibilities
Expect bugs and odd behavior, and consider it to be in forever beta
\b
Known bugs:
Programmes without clear season/episode labels might display odd names and numbers
PlutoTV does not feature the ability to request quality at the moment and defaults to best
TubiTV and PlutoTV does not work with --info at the moment
\b
Free streaming services are free for a reason and usually comes with gaps and odd labels
It's strongly recommended to use --titles to view episodes before downloading!
\b
Examples:
python freevine.py --titles URL
python freevine.py --episode S01E01 URL
python freevine.py --info --episode S01E01 URL
python freevine.py --episode S01E01-S01E10 URL
python freevine.py --quality 720p --season S01 URL
python freevine.py --remote --season S01 URL
\b
Download single episodes directly by URL:
python freevine.py EPISODE_URL
python freevine.py --quality EPISODE_URL
python freevine.py --info EPISODE_URL
"""

312
helpers/search.py Normal file
View File

@@ -0,0 +1,312 @@
import re
import uuid
import httpx
from rich.console import Console
from helpers.dict import _search
console = Console()
class Search:
def __init__(self, alias: str, keywords: str) -> None:
if alias:
alias = alias.upper()
if keywords:
keywords = keywords.lower()
self.client = httpx.Client(
headers={
"user-agent": (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/118.0.0.0 Safari/537.36"
),
}
)
self.alias = [alias]
self.keywords = keywords
if "," in self.alias[0]:
self.alias = [x for x in self.alias[0].split(",")]
self.services = _search(self.keywords)
def sanitize(title: str) -> str:
title = title.lower()
title = title.replace("&", "and")
title = re.sub(r"[:;/()]", "", title)
title = re.sub(r"[ ]", "-", title)
title = re.sub(r"[\\*!?¿,'\"<>|$#`]", "", title)
title = re.sub(rf"[{'.'}]{{2,}}", ".", title)
title = re.sub(rf"[{'_'}]{{2,}}", "_", title)
title = re.sub(rf"[{'-'}]{{2,}}", "-", title)
title = re.sub(rf"[{' '}]{{2,}}", " ", title)
return title
def search_get(search: object, service: dict):
url = service["url"]
params = service.get("params", {})
search.client.headers.update(service.get("header", {}))
cookies = service.get("collect", {})
return search.client.get(url, cookies=cookies, params=params).json()
def search_post(search: object, service: dict):
url = service["url"]
search.client.headers.update(service.get("header", {}))
payload = service.get("payload", {})
if service.get("token"):
token = search.client.get(service["token"]).json()["csrf"]
search.client.headers.update({"csrf-token": token})
return search.client.post(url, cookies=search.client.cookies, json=payload).json()
def parse_results(query: dict, service: dict, client=None):
template = """
[bold]{service}[/bold]
Title: {title}
Type: {type}
Synopsis: {synopsis}
Link: {url}
"""
results = []
if service["name"] == "BBC iPlayer":
for field in query["results"]:
results.append(
template.format(
service=service["name"],
title=field["title"],
synopsis=field["synopsis"],
type="programme" if field["type"] == "brand" else field["type"],
url=field["url"],
)
)
if service["name"] == "ALL4":
for field in query["results"]:
results.append(
template.format(
service=service["name"],
title=field["brand"]["title"],
synopsis=field["brand"]["description"],
type="",
url=field["brand"]["href"],
)
)
if service["name"] == "My5":
link = "https://www.channel5.com/show/"
for field in query["shows"]:
results.append(
template.format(
service=service["name"],
title=field["title"],
synopsis=field.get("s_desc"),
type=field.get("genre"),
url=f"{link}{field['f_name']}",
)
)
if service["name"] == "ITV":
link = "https://www.itv.com/watch"
for field in query["results"]:
special = field["data"].get("specialTitle")
standard = field["data"].get("programmeTitle")
film = field["data"].get("filmTitle")
title = special if special else standard if standard else film
slug = sanitize(title)
_id = field["data"]["legacyId"]["apiEncoded"]
_id = "_".join(_id.split("_")[:2]).replace("_", "a")
_id = re.sub(r"a000\d+", "", _id)
results.append(
template.format(
service=service["name"],
title=title,
synopsis=field["data"]["synopsis"],
type=field["entityType"],
url=f"{link}/{slug}/{_id}",
)
)
if service["name"] == "STV Player":
for field in query["records"]["page"]:
results.append(
template.format(
service=service["name"],
title=field["title"],
synopsis=field.get("resultDescriptionTx"),
type="programme",
url=field["url"],
)
)
if service["name"] == "CRACKLE":
link = "https://www.crackle.com/details"
for field in query["data"]["items"]:
results.append(
template.format(
service=service["name"],
title=field["metadata"][0]["title"],
synopsis=field["metadata"][0].get("longDescription"),
type=field.get("type"),
url=f"{link}/{field['id']}/{field['metadata'][0]['slug']}",
)
)
if service["name"] == "CTV":
link = "https://www.ctv.ca"
for field in query["data"]["searchMedia"]["page"]["items"]:
results.append(
template.format(
service=service["name"],
title=field["title"],
synopsis=None,
type=field["path"].split("/")[1],
url=f"{link}{field['path']}",
)
)
if service["name"] == "UKTV Play":
link = "https://uktvplay.co.uk/shows/{slug}/watch-online"
for field in query:
results.append(
template.format(
service=service["name"],
title=field["name"],
synopsis=field.get("synopsis"),
type=field.get("type"),
url=link.format(slug=field["slug"]),
)
)
if service["name"] == "PlutoTV":
params = {
"appName": "web",
"appVersion": "na",
"clientID": str(uuid.uuid1()),
"clientModelNumber": "na",
}
token = client.get("https://boot.pluto.tv/v4/start", params=params).json()[
"sessionToken"
]
client.headers.update({"Authorization": f"Bearer {token}"})
query = client.get(service["url"], params=service["params"]).json()
link = "https://pluto.tv/en/on-demand/{type}/{id}/details"
for field in query["data"]:
if "timeline" not in field["type"]:
results.append(
template.format(
service=service["name"],
title=field["name"],
synopsis=field.get("synopsis"),
type=field["type"],
url=link.format(
type="movies" if field["type"] == "movie" else "series",
id=field["id"],
),
)
)
if service["name"] == "The Roku Channel":
link = "https://therokuchannel.roku.com/details/{id}/{title}"
for field in query["view"]:
_desc = field["content"].get("descriptions")
results.append(
template.format(
service=service["name"],
title=field["content"]["title"],
synopsis=_desc["250"]["text"] if _desc.get("250") else None,
type=field["content"].get("type"),
url=link.format(
id=field["content"]["meta"]["id"],
title=sanitize(field["content"]["title"]),
),
)
)
if service["name"] == "TubiTV":
link = "https://tubitv.com/{type}/{id}/{title}"
for field in query:
type = (
"series"
if field["type"] == "s"
else "movies"
if field["type"] == "v"
else field["type"]
)
results.append(
template.format(
service=service["name"],
title=field["title"],
synopsis=field.get("description"),
type=type,
url=link.format(
type=type,
id=field["id"],
title=sanitize(field["title"]),
),
)
)
return results
def search_engine(alias: str, keywords: str):
search = Search(alias, keywords)
services = [
service
for service in search.services
if any(
i in service_alias
for i in search.alias
for service_alias in service["alias"]
)
]
queries = []
with console.status("Searching..."):
for service in services:
if service["method"] == "GET":
query = search_get(search, service)
if service["method"] == "POST":
query = search_post(search, service)
results = parse_results(query, service, search.client)
queries.append(results)
queries = [results for results in queries]
num_matches = 5 if len(services) >= 2 else 10
matches = [
[result for result in query if service["name"] in result][:num_matches]
for service in services
for query in queries
]
for match in matches:
for result in match:
console.print(result)

View File

@@ -1,6 +1,6 @@
import sys
import re
import importlib
import importlib.util
import sys
from pathlib import Path
@@ -9,42 +9,92 @@ from urllib.parse import urlparse
from helpers.utilities import info, error
def _services(domain: str):
supported_services = []
def _services():
services = Path("services")
services = list(Path("services").glob("*.py"))
for service in services:
srvc = Path(service).stem
if len(srvc) == 0 or srvc.startswith("_"):
continue
supported_services.append(srvc)
if domain not in supported_services:
error("Service not supported")
sys.exit(1)
supported_services = {
"www.bbc.co.uk": {
"name": "BBC",
"alias": "BBC iPlayer",
"path": services / "bbc.py",
},
"www.channel4.com": {
"name": "CHANNEL4",
"alias": "ALL4",
"path": services / "channel4.py",
},
"www.channel5.com": {
"name": "CHANNEL5",
"alias": "My5 TV",
"path": services / "channel5.py",
},
"www.crackle.com": {
"name": "CRACKLE",
"alias": "CRACKLE",
"path": services / "crackle.py",
},
"www.ctv.ca": {
"name": "CTV",
"alias": "CTV",
"path": services / "ctv.py",
},
"www.itv.com": {
"name": "ITV",
"alias": "ITVX",
"path": services / "itv.py",
},
"pluto.tv": {
"name": "PLUTO",
"alias": "PlutoTV",
"path": services / "pluto.py",
},
"therokuchannel.roku.com": {
"name": "ROKU",
"alias": "The Roku Channel",
"path": services / "roku.py",
},
"player.stv.tv": {
"name": "STV",
"alias": "STV Player",
"path": services / "stv.py",
},
"tubitv.com": {
"name": "TUBITV",
"alias": "TubiTV",
"path": services / "tubitv.py",
},
"uktvplay.co.uk": {
"name": "UKTVPLAY",
"alias": "UKTV Play",
"path": services / "uktvplay.py",
},
}
return supported_services
def get_service(url: str):
parse = urlparse(url)
netloc = parse.netloc.split(".")
supported = _services()
if len(netloc) == 4:
domain = netloc[1]
elif len(netloc) == 3 and netloc[2] == "uk":
domain = netloc[0]
elif len(netloc) == 3:
domain = netloc[1]
else:
domain = netloc[0]
find_service = next(
(
info
for service, info in supported.items()
if service == urlparse(url).netloc
),
None,
)
services = _services(domain)
if find_service is None:
error("Service is not supported")
sys.exit(1)
for service in services:
if service == domain:
service_module = importlib.import_module("services." + service)
srvc = getattr(service_module, service.upper())
info(srvc.__name__)
return srvc
spec = importlib.util.spec_from_file_location(
find_service["name"], str(find_service["path"])
)
service_module = importlib.util.module_from_spec(spec)
sys.modules[find_service["name"]] = service_module
spec.loader.exec_module(service_module)
srvc = getattr(service_module, find_service["name"])
info(find_service["alias"])
return srvc

View File

@@ -137,6 +137,31 @@ class UKTVPLAY(Config):
return closest_match, pssh
return heights[0], pssh
def get_content(self, url: str) -> object:
if self.movie:
with self.console.status("Fetching titles..."):
content = self.get_movies(self.url)
title = string_cleaning(str(content))
info(f"{str(content)}\n")
else:
with self.console.status("Fetching titles..."):
content = self.get_series(url)
for episode in content:
episode.name = episode.get_filename()
title = string_cleaning(str(content))
seasons = Counter(x.season for x in content)
num_seasons = len(seasons)
num_episodes = sum(seasons.values())
info(
f"{str(content)}: {num_seasons} Season(s), {num_episodes} Episode(s)\n"
)
return content, title
def get_episode_from_url(self, url: str):
html = self.client.get(url).text