From ffc0cc8a8c21c169dd70615300799cd84d8c1ac6 Mon Sep 17 00:00:00 2001 From: Boris-Chengbiao Zhou Date: Sun, 4 Jun 2017 19:56:05 +0200 Subject: [PATCH] Split into Presentation into separate module --- src/main.rs | 272 +------------------------------------------- src/presentation.rs | 270 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 276 insertions(+), 266 deletions(-) create mode 100644 src/presentation.rs diff --git a/src/main.rs b/src/main.rs index 7254b27..593b785 100644 --- a/src/main.rs +++ b/src/main.rs @@ -8,24 +8,20 @@ extern crate reqwest; extern crate zip; mod catalog; +mod presentation; use std::env; -use std::process::Command; use std::collections::HashMap; -use std::fs::File; -use std::io::{self, Read, BufReader, Write}; -use std::path::{Path, PathBuf}; -use std::time::{Instant, Duration}; +use std::io::{self, Read}; +use std::path::Path; use std::sync::Mutex; use clap::{App, Arg}; -use json::JsonValue; -use reqwest::{Client, Response, RedirectPolicy, RequestBuilder, StatusCode}; -use reqwest::header::{Cookie, Charset, SetCookie}; -use reqwest::header::{ContentDisposition, DispositionType, DispositionParam}; -use zip::ZipArchive; +use reqwest::{Client, Response, RedirectPolicy, RequestBuilder}; +use reqwest::header::{Cookie, SetCookie}; use catalog::*; +use presentation::Presentation; // TODO: Old "Publish To Go" packages get sweeped from the server and you have to request a new one; // Implement this @@ -54,180 +50,6 @@ enum DownloadError { AuthorizationTimeout, } -#[derive(Debug)] -enum Presentation { - PublishToGo { name: String, download_url: String }, - VideoOnDemand { - name: String, - resource_id: String, - query_string: String, - }, -} - -impl Presentation { - fn name(&self) -> &str { - match *self { - Presentation::PublishToGo { ref name, .. } | - Presentation::VideoOnDemand { ref name, .. } => name, - } - } - - fn download(&self, out_dir: &Path) -> Result<(), DownloadError> { - fn download_to_file_and_check bool>(response: &mut Response, - out_file: &Path, - check: F) - -> Result<(), io::Error> { - println!("Output file: {}", out_file.display()); - if out_file.exists() { - println!("File is present already. Checking validity..."); - if check(out_file) { - println!("Valid file! Skipping!"); - return Ok(()); - } else { - println!("Corrupt file! Starting anew!"); - } - // TODO?: We could try resuming the download if the server supports it - // http://stackoverflow.com/questions/41444297/ - // http://stackoverflow.com/questions/3428102/ - } - download_to_file(response, out_file)?; - assert!(check(out_file)); - Ok(()) - } - - match *self { - Presentation::PublishToGo { ref download_url, .. } => { - let mut response = try_to_get_valid_response(|client| { - client - .get(download_url) - .header(construct_cookie()) - }, - |resp| { - resp.headers().get::().is_some() - }); - - let filename = { - let content_disposition = - response.headers().get::().unwrap(); - assert_eq!(content_disposition.disposition, DispositionType::Attachment); - let mut name = None; - for param in &content_disposition.parameters { - match *param { - DispositionParam::Filename(ref charset, _, ref vec) => { - assert_eq!(&Charset::Ext("UTF-8".to_string()), charset); - name = Some(String::from_utf8(vec.to_vec()) - .expect("Suggested filename isn't valid UTF-8!",)); - } - _ => continue, - } - } - name.expect("Missing filename in ContentDisposition!") - }; - - let mut path = PathBuf::from(out_dir); - path.push(fix_filename(&filename)); - - download_to_file_and_check(&mut response, &path, |path| zip_is_valid(path)) - .map_err(|e| DownloadError::IoError(e)) - } - Presentation::VideoOnDemand { - ref resource_id, - ref query_string, - .. - } => { - let videos = { - let payload = format!("{{\"getPlayerOptionsRequest\":{{\ - \"ResourceId\":\"{}\",\ - \"QueryString\":\"{}\"\ - }}}}", - resource_id, - query_string); - - let url = "https://streams.tum.de/Mediasite/PlayerService/\ - PlayerService.svc/json/GetPlayerOptions"; - let mut res = try_to_get_response(|client| { - client - .post(url) - .header(construct_cookie()) - .header(reqwest::header::ContentType::json()) - .body(&*payload) - }); - - let json_text = read_response_body(&mut res); - if json_text.contains("You are not authorized to view the requested content") { - return Err(DownloadError::AuthorizationTimeout); - } - let json = json::parse(&json_text).expect("Failed parsing the json!"); - - let mut vec = Vec::new(); - - for stream in json["d"]["Presentation"]["Streams"].members() { - assert_eq!(stream["VideoUrls"].len(), 1); - vec.push(stream["VideoUrls"][0]["Location"].to_string()); - } - vec - }; - for (i, download_url) in videos.iter().enumerate() { - let mut response = try_to_get_valid_response(|client| { - client.get(download_url).header(construct_cookie()) - }, - |resp| { - resp.status() == - &StatusCode::Ok - }); - // TODO: Support formats besides mp4 - // extract extension from url or somewehre else... - let filename = format!("{} - {}.mp4", fix_filename(self.name()), i + 1); - - let mut path = PathBuf::from(out_dir); - path.push(fix_filename(&filename)); - - let mut path = PathBuf::from(out_dir); - path.push(fix_filename(&filename)); - - download_to_file_and_check(&mut response, &path, |path| check_video(path)) - .map_err(|e| DownloadError::IoError(e))?; - } - Ok(()) - } - } - } -} - -impl<'a> From<&'a JsonValue> for Presentation { - fn from(json: &'a JsonValue) -> Presentation { - if json["DownloadUrls"].len() >= 1 { - assert_eq!(json["DownloadUrls"].len(), 1); - Presentation::PublishToGo { - name: json["Name"] - .clone() - .take_string() - .expect("json: Presentation didn't contain a \"Name\"!"), - download_url: json["DownloadUrls"][0]["Url"] - .clone() - .take_string() - .expect("json: Presentation didn't have a download url!"), - } - } else { - // No "Publish To Go" -> download video streams manually - let player_url = json["PlayerUrl"].to_string(); - - let index = player_url.find("?catalog=").unwrap(); - let index_end_query = index + "?catalog=499c5d25-37a6-4853-95d5-bfbf7f76fcf2".len(); - let index_start_resource = index - "cdf23465581b45aa92d588ebd59619d91d".len(); - - Presentation::VideoOnDemand { - name: json["Name"] - .clone() - .take_string() - .expect("json: Presentation didn't contain a \"Name\"!"), - query_string: player_url[index..index_end_query].to_string(), - resource_id: player_url[index_start_resource..index].to_string(), - } - } - } -} - fn main() { let matches = App::new("TumMediasiteDownloader") .author("Boris-Chengbiao Zhou ") @@ -376,59 +198,6 @@ fn get_catalog_id(name: &str) -> String { body[(idx + pre_len)..(idx + pre_len + len)].to_string() } -fn download_to_file(response: &mut Response, path: &Path) -> Result<(), io::Error> { - let mut file = File::create(&path).expect("Failed to create file!"); - - let mut reader = BufReader::new(response); - let mut buf = [0u8; 8 * 1024]; - - let update_duration_millis = 1000; - let mut last = Instant::now(); - let mut done = 0; - - loop { - let len = match reader.read(&mut buf) { - Ok(0) => break, - Ok(len) => len, - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, - Err(e) => return Err(e), - }; - - file.write_all(&buf[0..len]) - .expect("Failed writing into the file!"); - done += len; - - let now = Instant::now(); - if now - last > Duration::from_millis(update_duration_millis) { - print!("{:.*} kB/s \r", - 2, - done as f64 * 1000.0 / update_duration_millis as f64 / 1024.0); - std::io::stdout() - .flush() - .expect("Failed flushing the terminal!"); - last = now; - done = 0; - } - } - - println!("Download successful!"); - - Ok(()) -} - -fn fix_filename(string: &str) -> String { - string - .replace('/', "_") - .replace('\\', "_") - .replace(':', " -") - .replace('*', "%2A") - .replace('?', "%3F") - .replace('"', "'") - .replace('<', "'") - .replace('>', "'") - .replace('|', "_") -} - fn json_to_presentations(json_str: &str) -> Vec { let mut vec = Vec::new(); let mut json = json::parse(json_str).expect("Failed parsing the json!"); @@ -462,35 +231,6 @@ fn construct_cookie() -> Cookie { Cookie(vec![format!("MediasiteAuth={}", *AUTH.lock().unwrap())]) } -fn zip_is_valid(path: &Path) -> bool { - assert!(path.exists()); - assert!(path.is_file()); - let file = File::open(path).expect("Failed opening the zip file!"); - ZipArchive::new(file).is_ok() -} - -fn check_video(path: &Path) -> bool { - let command = Command::new("ffmpeg") - .arg("-v") - .arg("error") - .arg("-i") - .arg(path) - .arg("-f") - .arg("null") - .arg("-") - .output(); - - match command { - Ok(output) => output.stderr.is_empty(), - Err(_) => { - println!("Failed to check {} with ffmpeg. Perhaps ffmpeg isn't in your path. \ - Skipping check...", - path.display()); - true - } - } -} - fn try_to_get_response(f: F) -> Response where F: Fn(&Client) -> RequestBuilder { diff --git a/src/presentation.rs b/src/presentation.rs new file mode 100644 index 0000000..11cfb81 --- /dev/null +++ b/src/presentation.rs @@ -0,0 +1,270 @@ +use std::process::Command; +use std::fs::File; +use std::io::{self, Read, BufReader, Write}; +use std::path::{Path, PathBuf}; +use std::time::{Instant, Duration}; + +use json::{self, JsonValue}; +use reqwest::{Response, StatusCode}; +use reqwest::header::Charset; +use reqwest::header::{ContentType, ContentDisposition, DispositionType, DispositionParam}; +use zip::ZipArchive; + +use {try_to_get_response, try_to_get_valid_response, construct_cookie, read_response_body}; +use DownloadError; + +#[derive(Debug)] +pub enum Presentation { + PublishToGo { name: String, download_url: String }, + VideoOnDemand { + name: String, + resource_id: String, + query_string: String, + }, +} + +impl Presentation { + pub(crate) fn name(&self) -> &str { + match *self { + Presentation::PublishToGo { ref name, .. } | + Presentation::VideoOnDemand { ref name, .. } => name, + } + } + + pub(crate) fn download(&self, out_dir: &Path) -> Result<(), DownloadError> { + fn download_to_file_and_check bool>(response: &mut Response, + out_file: &Path, + check: F) + -> Result<(), io::Error> { + println!("Output file: {}", out_file.display()); + if out_file.exists() { + println!("File is present already. Checking validity..."); + if check(out_file) { + println!("Valid file! Skipping!"); + return Ok(()); + } else { + println!("Corrupt file! Starting anew!"); + } + // TODO?: We could try resuming the download if the server supports it + // http://stackoverflow.com/questions/41444297/ + // http://stackoverflow.com/questions/3428102/ + } + download_to_file(response, out_file)?; + assert!(check(out_file)); + Ok(()) + } + + match *self { + Presentation::PublishToGo { ref download_url, .. } => { + let mut response = try_to_get_valid_response(|client| { + client + .get(download_url) + .header(construct_cookie()) + }, + |resp| { + resp.headers().get::().is_some() + }); + + let filename = { + let content_disposition = + response.headers().get::().unwrap(); + assert_eq!(content_disposition.disposition, DispositionType::Attachment); + let mut name = None; + for param in &content_disposition.parameters { + match *param { + DispositionParam::Filename(ref charset, _, ref vec) => { + assert_eq!(&Charset::Ext("UTF-8".to_string()), charset); + name = Some(String::from_utf8(vec.to_vec()) + .expect("Suggested filename isn't valid UTF-8!",)); + } + _ => continue, + } + } + name.expect("Missing filename in ContentDisposition!") + }; + + let mut path = PathBuf::from(out_dir); + path.push(fix_filename(&filename)); + + download_to_file_and_check(&mut response, &path, |path| check_zip(path)) + .map_err(|e| DownloadError::IoError(e)) + } + Presentation::VideoOnDemand { + ref resource_id, + ref query_string, + .. + } => { + let videos = { + let payload = format!("{{\"getPlayerOptionsRequest\":{{\ + \"ResourceId\":\"{}\",\ + \"QueryString\":\"{}\"\ + }}}}", + resource_id, + query_string); + + let url = "https://streams.tum.de/Mediasite/PlayerService/\ + PlayerService.svc/json/GetPlayerOptions"; + let mut res = try_to_get_response(|client| { + client + .post(url) + .header(construct_cookie()) + .header(ContentType::json()) + .body(&*payload) + }); + + let json_text = read_response_body(&mut res); + if json_text.contains("You are not authorized to view the requested content") { + return Err(DownloadError::AuthorizationTimeout); + } + let json = json::parse(&json_text).expect("Failed parsing the json!"); + + let mut vec = Vec::new(); + + for stream in json["d"]["Presentation"]["Streams"].members() { + assert_eq!(stream["VideoUrls"].len(), 1); + vec.push(stream["VideoUrls"][0]["Location"].to_string()); + } + vec + }; + for (i, download_url) in videos.iter().enumerate() { + let mut response = try_to_get_valid_response(|client| { + client.get(download_url).header(construct_cookie()) + }, + |resp| { + resp.status() == + &StatusCode::Ok + }); + // TODO: Support formats besides mp4 + // extract extension from url or somewehre else... + let filename = format!("{} - {}.mp4", fix_filename(self.name()), i + 1); + + let mut path = PathBuf::from(out_dir); + path.push(fix_filename(&filename)); + + let mut path = PathBuf::from(out_dir); + path.push(fix_filename(&filename)); + + download_to_file_and_check(&mut response, &path, |path| check_video(path)) + .map_err(|e| DownloadError::IoError(e))?; + } + Ok(()) + } + } + } +} + +impl<'a> From<&'a JsonValue> for Presentation { + fn from(json: &'a JsonValue) -> Presentation { + if json["DownloadUrls"].len() >= 1 { + assert_eq!(json["DownloadUrls"].len(), 1); + Presentation::PublishToGo { + name: json["Name"] + .clone() + .take_string() + .expect("json: Presentation didn't contain a \"Name\"!"), + download_url: json["DownloadUrls"][0]["Url"] + .clone() + .take_string() + .expect("json: Presentation didn't have a download url!"), + } + } else { + // No "Publish To Go" -> download video streams manually + let player_url = json["PlayerUrl"].to_string(); + + let index = player_url.find("?catalog=").unwrap(); + let index_end_query = index + "?catalog=499c5d25-37a6-4853-95d5-bfbf7f76fcf2".len(); + let index_start_resource = index - "cdf23465581b45aa92d588ebd59619d91d".len(); + + Presentation::VideoOnDemand { + name: json["Name"] + .clone() + .take_string() + .expect("json: Presentation didn't contain a \"Name\"!"), + query_string: player_url[index..index_end_query].to_string(), + resource_id: player_url[index_start_resource..index].to_string(), + } + } + } +} + +fn download_to_file(response: &mut Response, path: &Path) -> Result<(), io::Error> { + let mut file = File::create(&path).expect("Failed to create file!"); + + let mut reader = BufReader::new(response); + let mut buf = [0u8; 8 * 1024]; + + let update_duration_millis = 1000; + let mut last = Instant::now(); + let mut done = 0; + + loop { + let len = match reader.read(&mut buf) { + Ok(0) => break, + Ok(len) => len, + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; + + file.write_all(&buf[0..len]) + .expect("Failed writing into the file!"); + done += len; + + let now = Instant::now(); + if now - last > Duration::from_millis(update_duration_millis) { + print!("{:.*} kB/s \r", + 2, + done as f64 * 1000.0 / update_duration_millis as f64 / 1024.0); + ::std::io::stdout() + .flush() + .expect("Failed flushing the terminal!"); + last = now; + done = 0; + } + } + + println!("Download successful!"); + + Ok(()) +} + +fn fix_filename(string: &str) -> String { + string + .replace('/', "_") + .replace('\\', "_") + .replace(':', " -") + .replace('*', "%2A") + .replace('?', "%3F") + .replace('"', "'") + .replace('<', "'") + .replace('>', "'") + .replace('|', "_") +} + +fn check_zip(path: &Path) -> bool { + assert!(path.exists()); + assert!(path.is_file()); + let file = File::open(path).expect("Failed opening the zip file!"); + ZipArchive::new(file).is_ok() +} + +fn check_video(path: &Path) -> bool { + let command = Command::new("ffmpeg") + .arg("-v") + .arg("error") + .arg("-i") + .arg(path) + .arg("-f") + .arg("null") + .arg("-") + .output(); + + match command { + Ok(output) => output.stderr.is_empty(), + Err(_) => { + println!("Failed to check {} with ffmpeg. Perhaps ffmpeg isn't in your path. \ + Skipping check...", + path.display()); + true + } + } +}