bobo1239/TumMediasiteDownloader
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Add support for presentations without PublishToGo

Browse Source
This commit is contained in:
Boris-Chengbiao Zhou
2017-05-12 02:18:38 +02:00
parent 2826a83396
commit 3c3bf8554f
3 changed files with 234 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

285
src/main.rs
View File

@@ -8,6 +8,7 @@ extern crate reqwest;
extern crate zip;
use std::env;
use std::process::Command;
use std::collections::HashMap;
use std::fs::File;
use std::io::{self, Read, BufReader, Write};
@@ -16,14 +17,17 @@ use std::time::{Instant, Duration};
use clap::{App, Arg};
use json::JsonValue;
use reqwest::{Client, Response, RedirectPolicy, RequestBuilder};
use reqwest::{Client, Response, RedirectPolicy, RequestBuilder, StatusCode};
use reqwest::header::{Cookie, Charset, SetCookie};
use reqwest::header::{ContentDisposition, DispositionType, DispositionParam};
use zip::ZipArchive;
use catalogs::*;
#[allow(dead_code)]
// TODO: Old "Publish To Go" packages get sweeped from the server and you have to request a new one;
// Implement this
// TODO: sometimes you need to access the video listing at least once using moodle; emulate this
mod catalogs {
type Login = Option<(&'static str, &'static str)>;
type CatalogDef = (&'static str, Login);
@@ -34,8 +38,8 @@ mod catalogs {
pub const ERA: CatalogDef = ("era-2016", None);
pub const GAD: CatalogDef = ("17s-grundlagen-algorithmen-und-datenstrukturen", None);
pub const EIDS: CatalogDef = ("TODO", None);
pub const LA: CatalogDef = ("17s-lineare-algebra-fuer-informatik", None);
// EIST is on a Nextcloud
}
const MAX_RETRIES: u8 = 10;
@@ -50,22 +54,147 @@ lazy_static! {
}
#[derive(Debug)]
struct Presentation {
name: String,
download_url: String,
enum Presentation {
PublishToGo {
name: String,
download_url: String,
},
VideoOnDemand {
name: String,
resource_id: String,
query_string: String,
}
}
impl Presentation {
fn name(&self) -> &str {
match *self {
Presentation::PublishToGo {ref name, ..} |
Presentation::VideoOnDemand {ref name, ..} => {name},
}
}
fn download(&self, presentation: &Presentation, out_dir: &Path, auth: &str) {
fn download_to_file_and_check<F: Fn(&Path) -> bool>(response: &mut Response, out_file: &Path, check: F) {
println!("Output file: {}", out_file.display());
if out_file.exists() {
if check(out_file) {
println!("Already present and valid! Skipping!");
return;
} else {
println!("Found corrupt file! Starting anew!");
}
// TODO?: We could try resuming the download if the server supports it
// http://stackoverflow.com/questions/41444297/
// http://stackoverflow.com/questions/3428102/
}
let file = download_to_file(response, &out_file);
::std::mem::drop(file);
assert!(check(out_file));
}
match *self {
Presentation::PublishToGo {ref download_url, ..} => {
let mut response =
try_to_get_valid_response(|client| {
client.get(download_url)
.header(construct_cookie(auth))
},
|resp| resp.headers().get::<ContentDisposition>().is_some());
let filename = {
let content_disposition = response.headers().get::<ContentDisposition>().unwrap();
assert_eq!(content_disposition.disposition, DispositionType::Attachment);
let mut name = None;
for param in &content_disposition.parameters {
match *param {
DispositionParam::Filename(ref charset, _, ref vec) => {
assert_eq!(&Charset::Ext("UTF-8".to_string()), charset);
name = Some(String::from_utf8(vec.to_vec())
.expect("Suggested filename isn't valid UTF-8!"));
}
_ => continue,
}
}
name.expect("Missing filename in ContentDisposition!")
};
let mut path = PathBuf::from(out_dir);
path.push(fix_filename(&filename));
download_to_file_and_check(&mut response, &path, |path| zip_is_valid(path));
},
Presentation::VideoOnDemand {ref resource_id, ref query_string, ..} => {
let videos = {
let payload = format!("{{\"getPlayerOptionsRequest\":{{\
\"ResourceId\":\"{}\",\
\"QueryString\":\"{}\"\
}}}}", resource_id, query_string);
let url = "https://streams.tum.de/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions";
let mut res = try_to_get_response(|client| client.post(url).header(construct_cookie(auth)).header(reqwest::header::ContentType::json()).body(&*payload));
let json_text = read_response_body(&mut res);
let json = json::parse(&json_text).expect("Failed parsing the json!");
let mut vec = Vec::new();
for stream in json["d"]["Presentation"]["Streams"].members() {
assert_eq!(stream["VideoUrls"].len(), 1);
vec.push(stream["VideoUrls"][0]["Location"].to_string());
}
vec
};
for (i, download_url) in videos.iter().enumerate() {
let mut response = try_to_get_valid_response(|client| {
client.get(download_url)
.header(construct_cookie(auth))
},
|resp| resp.status() == &StatusCode::Ok);
// TODO: Support formats besides mp4; extract extension from url or somewehre else...
let filename = format!("{} - {}.mp4", fix_filename(presentation.name()), i + 1);
let mut path = PathBuf::from(out_dir);
path.push(fix_filename(&filename));
let mut path = PathBuf::from(out_dir);
path.push(fix_filename(&filename));
download_to_file_and_check(&mut response, &path, |path| check_video(path));
}
},
}
}
}
impl<'a> From<&'a JsonValue> for Presentation {
fn from(json: &'a JsonValue) -> Presentation {
Presentation {
name: json["Name"]
.clone()
.take_string()
.expect("json: Presentation didn't contain a \"Name\"!"),
download_url: json["DownloadUrls"][0]["Url"]
.clone()
.take_string()
.expect("json: Presentation didn't have a download url!"),
if json["DownloadUrls"].len() >= 1 {
assert_eq!(json["DownloadUrls"].len(), 1);
Presentation::PublishToGo {
name: json["Name"]
.clone()
.take_string()
.expect("json: Presentation didn't contain a \"Name\"!"),
download_url: json["DownloadUrls"][0]["Url"]
.clone()
.take_string()
.expect("json: Presentation didn't have a download url!"),
}
} else {
// No "Publish To Go" -> download video streams manually
let player_url = json["PlayerUrl"].to_string();
let index = player_url.find("?catalog=").unwrap();
let index_end_query = index + "?catalog=499c5d25-37a6-4853-95d5-bfbf7f76fcf2".len();
let index_start_resource = index - "cdf23465581b45aa92d588ebd59619d91d".len();
Presentation::VideoOnDemand {
name: json["Name"].clone().take_string().expect("json: Presentation didn't contain a \"Name\"!"),
query_string: player_url[index..index_end_query].to_string(),
resource_id: player_url[index_start_resource..index].to_string(),
}
}
}
}
@@ -101,7 +230,6 @@ fn main() {
"EIDI" => EIDI,
"ERA" => ERA,
"GAD" => GAD,
"EIDS" => EIDS,
"LA" => LA,
n => {
if let Some(username) = matches.value_of("username") {
@@ -181,8 +309,8 @@ fn download_catalog(catalog_name: &str, out_dir: &Path, auth: &str) {
println!("Downloading {}/{}: {}",
i + 1,
presentations.len(),
presentation.name);
download_presentation(presentation, out_dir, auth);
presentation.name());
presentation.download(presentation, out_dir, auth);
}
}
@@ -204,78 +332,39 @@ fn get_catalog_id(name: &str, auth: &str) -> String {
body[(idx + pre_len)..(idx + pre_len + len)].to_string()
}
fn download_presentation(presentation: &Presentation, out_dir: &Path, auth: &str) {
let response =
try_to_get_valid_response(|client| {
client.get(&presentation.download_url)
.header(construct_cookie(auth))
},
|resp| resp.headers().get::<ContentDisposition>().is_some());
fn download_to_file(response: &mut Response, path: &Path) -> File {
let mut file = File::create(&path).expect("Failed to create file!");
let filename = {
let content_disposition = response.headers().get::<ContentDisposition>().unwrap();
assert_eq!(content_disposition.disposition, DispositionType::Attachment);
let mut name = None;
for param in &content_disposition.parameters {
match *param {
DispositionParam::Filename(ref charset, _, ref vec) => {
assert_eq!(&Charset::Ext("UTF-8".to_string()), charset);
name = Some(String::from_utf8(vec.to_vec())
.expect("Suggested filename isn't valid UTF-8!"));
}
_ => continue,
let mut reader = BufReader::new(response);
let mut buf = [0u8; 8 * 1024];
let update_duration_millis = 1000;
let mut last = Instant::now();
let mut done = 0;
loop {
let len = match reader.read(&mut buf) {
Ok(0) => break,
Ok(len) => len,
Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => panic!(e),
};
file.write_all(&buf[0..len]).expect("Failed writing into the file!");
done += len;
let now = Instant::now();
if now - last > Duration::from_millis(update_duration_millis) {
print!("{:.*} kB/s \r",
2,
done as f64 * 1000.0 / update_duration_millis as f64 / 1024.0);
std::io::stdout().flush().expect("Failed flushing the terminal!");
last = now;
done = 0;
}
}
name.expect("Missing filename in ContentDisposition!")
};
let mut path = PathBuf::from(out_dir);
path.push(fix_filename(&filename));
println!("Output file: {}", path.display());
if path.exists() {
if zip_is_valid(&path) {
println!("Already present and valid! Skipping!");
return;
} else {
println!("Found corrupt zip! Starting anew!");
// TODO?: We could try resuming the download if the server supports it
// http://stackoverflow.com/questions/41444297/
// http://stackoverflow.com/questions/3428102/
}
}
let mut file = File::create(&path).expect("Failed to create file!");
let mut reader = BufReader::new(response);
let mut buf = [0u8; 8 * 1024];
let update_duration_millis = 1000;
let mut last = Instant::now();
let mut done = 0;
loop {
let len = match reader.read(&mut buf) {
Ok(0) => break,
Ok(len) => len,
Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => panic!(e),
};
file.write_all(&buf[0..len]).expect("Failed writing into the file!");
done += len;
let now = Instant::now();
if now - last > Duration::from_millis(update_duration_millis) {
print!("{:.*} kB/s\r",
2,
done as f64 * 1000.0 / update_duration_millis as f64 / 1024.0);
std::io::stdout().flush().expect("Failed flushing the terminal!");
last = now;
done = 0;
}
}
::std::mem::drop(file);
assert!(zip_is_valid(&path));
file
}
fn fix_filename(string: &str) -> String {
@@ -295,7 +384,6 @@ fn json_to_presentations(json_str: &str) -> Vec<Presentation> {
let mut json = json::parse(json_str).expect("Failed parsing the json!");
let mut count = 0;
for presentation in json["PresentationDetailsList"].members_mut() {
assert_eq!(1, presentation["DownloadUrls"].len());
vec.push(Presentation::from(&*presentation));
count += 1;
}
@@ -330,6 +418,29 @@ fn zip_is_valid(path: &Path) -> bool {
ZipArchive::new(file).is_ok()
}
fn check_video(path: &Path) -> bool {
let command = Command::new("ffmpeg")
.arg("-v")
.arg("error")
.arg("-i")
.arg(path)
.arg("-f")
.arg("null")
.arg("-")
.output();
match command {
Ok(output) => {
output.stderr.is_empty()
}
Err(_) => {
println!("Failed to check {} with ffmpeg. Perhaps ffmpeg isn't in your path. \
Skipping check...", path.display());
true
}
}
}
fn try_to_get_response<F>(f: F) -> Response
where F: Fn(&Client) -> RequestBuilder
{