From 51e4f591d6ad847306694fd17fccb4bcaa53763e Mon Sep 17 00:00:00 2001 From: Boris-Chengbiao Zhou Date: Mon, 15 May 2017 12:42:15 +0200 Subject: [PATCH] Make AUTH a global variable and refresh authorization when our previous one timed out; Re-enable ffmpeg check --- Cargo.lock | 24 ++-- src/main.rs | 317 +++++++++++++++++++++++++++++++--------------------- 2 files changed, 204 insertions(+), 137 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index cbfa584..d7da9a3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -67,7 +67,7 @@ name = "backtrace-sys" version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.46 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -103,7 +103,7 @@ name = "bzip2-sys" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.46 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -218,7 +218,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] name = "gcc" -version = "0.3.45" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] @@ -347,7 +347,7 @@ name = "miniz-sys" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.46 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", ] @@ -366,7 +366,7 @@ name = "native-tls" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "openssl 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl 0.9.12 (registry+https://github.com/rust-lang/crates.io-index)", "schannel 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)", "security-framework 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", "security-framework-sys 0.1.14 (registry+https://github.com/rust-lang/crates.io-index)", @@ -388,22 +388,22 @@ dependencies = [ [[package]] name = "openssl" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ "bitflags 0.8.2 (registry+https://github.com/rust-lang/crates.io-index)", "foreign-types 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "lazy_static 0.2.8 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", - "openssl-sys 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)", + "openssl-sys 0.9.12 (registry+https://github.com/rust-lang/crates.io-index)", ] [[package]] name = "openssl-sys" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ - "gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)", + "gcc 0.3.46 (registry+https://github.com/rust-lang/crates.io-index)", "gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "libc 0.2.22 (registry+https://github.com/rust-lang/crates.io-index)", "pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)", @@ -763,7 +763,7 @@ dependencies = [ "checksum error-chain 0.10.0 (registry+https://github.com/rust-lang/crates.io-index)" = "d9435d864e017c3c6afeac1654189b06cdb491cf2ff73dbf0d73b0f292f42ff8" "checksum flate2 0.2.19 (registry+https://github.com/rust-lang/crates.io-index)" = "36df0166e856739905cd3d7e0b210fe818592211a008862599845e012d8d304c" "checksum foreign-types 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "3e4056b9bd47f8ac5ba12be771f77a0dae796d1bbaaf5fd0b9c2d38b69b8a29d" -"checksum gcc 0.3.45 (registry+https://github.com/rust-lang/crates.io-index)" = "40899336fb50db0c78710f53e87afc54d8c7266fb76262fecc78ca1a7f09deae" +"checksum gcc 0.3.46 (registry+https://github.com/rust-lang/crates.io-index)" = "181e3cebba1d663bd92eb90e2da787e10597e027eb00de8d742b260a7850948f" "checksum gdi32-sys 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "0912515a8ff24ba900422ecda800b52f4016a56251922d397c576bf92c690518" "checksum httparse 1.2.2 (registry+https://github.com/rust-lang/crates.io-index)" = "77f756bed9ee3a83ce98774f4155b42a31b787029013f3a7d83eca714e500e21" "checksum hyper 0.10.10 (registry+https://github.com/rust-lang/crates.io-index)" = "36e108e0b1fa2d17491cbaac4bc460dc0956029d10ccf83c913dd0e5db3e7f07" @@ -785,8 +785,8 @@ dependencies = [ "checksum native-tls 0.1.2 (registry+https://github.com/rust-lang/crates.io-index)" = "1e94a2fc65a44729fe969cc973da87c1052ae3f000b2cb33029f14aeb85550d5" "checksum num-traits 0.1.37 (registry+https://github.com/rust-lang/crates.io-index)" = "e1cbfa3781f3fe73dc05321bed52a06d2d491eaa764c52335cf4399f046ece99" "checksum num_cpus 1.4.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ca313f1862c7ec3e0dfe8ace9fa91b1d9cb5c84ace3d00f5ec4216238e93c167" -"checksum openssl 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "241bcf67b1bb8d19da97360a925730bdf5b6176d434ab8ded55b4ca632346e3a" -"checksum openssl-sys 0.9.11 (registry+https://github.com/rust-lang/crates.io-index)" = "e5e0fd64cb2fa018ed2e7b2c8d9649114fe5da957c9a67432957f01e5dcc82e9" +"checksum openssl 0.9.12 (registry+https://github.com/rust-lang/crates.io-index)" = "bb5d1663b73d10c6a3eda53e2e9d0346f822394e7b858d7257718f65f61dfbe2" +"checksum openssl-sys 0.9.12 (registry+https://github.com/rust-lang/crates.io-index)" = "3a5886d87d3e2a0d890bf62dc8944f5e3769a405f7e1e9ef6e517e47fd7a0897" "checksum pkg-config 0.3.9 (registry+https://github.com/rust-lang/crates.io-index)" = "3a8b4c6b8165cd1a1cd4b9b120978131389f64bdaf456435caa41e630edba903" "checksum podio 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "e5422a1ee1bc57cc47ae717b0137314258138f38fd5f3cea083f43a9725383a0" "checksum quote 0.3.15 (registry+https://github.com/rust-lang/crates.io-index)" = "7a6e920b65c65f10b2ae65c831a81a073a89edd28c7cce89475bff467ab4167a" diff --git a/src/main.rs b/src/main.rs index e6365dc..7631b0d 100644 --- a/src/main.rs +++ b/src/main.rs @@ -14,6 +14,7 @@ use std::fs::File; use std::io::{self, Read, BufReader, Write}; use std::path::{Path, PathBuf}; use std::time::{Instant, Duration}; +use std::sync::Mutex; use clap::{App, Arg}; use json::JsonValue; @@ -27,6 +28,8 @@ use catalogs::*; // TODO: Old "Publish To Go" packages get sweeped from the server and you have to request a new one; // Implement this // TODO: sometimes you need to access the video listing at least once using moodle; emulate this +// TODO: Add aliases with semester postfix so other can contribute aliases; move to separate file +// split into files in general? mod catalogs { type Login = Option<(&'static str, &'static str)>; @@ -51,60 +54,73 @@ lazy_static! { client.redirect(RedirectPolicy::none()); client }; + + static ref USERNAME: Mutex = Mutex::new(String::new()); + static ref PASSWORD: Mutex = Mutex::new(String::new()); + static ref AUTH: Mutex = Mutex::new(String::new()); +} + +#[derive(Debug)] +enum DownloadError { + IoError(io::Error), + AuthorizationTimeout, } #[derive(Debug)] enum Presentation { - PublishToGo { - name: String, - download_url: String, - }, + PublishToGo { name: String, download_url: String }, VideoOnDemand { name: String, resource_id: String, query_string: String, - } + }, } impl Presentation { fn name(&self) -> &str { match *self { - Presentation::PublishToGo {ref name, ..} | - Presentation::VideoOnDemand {ref name, ..} => {name}, + Presentation::PublishToGo { ref name, .. } | + Presentation::VideoOnDemand { ref name, .. } => name, } } - fn download(&self, presentation: &Presentation, out_dir: &Path, auth: &str) { - fn download_to_file_and_check bool>(response: &mut Response, out_file: &Path, check: F) { - - println!("Output file: {}", out_file.display()); - if out_file.exists() { - if check(out_file) { - println!("Already present and valid! Skipping!"); - return; - } else { - println!("Found corrupt file! Starting anew!"); - } - // TODO?: We could try resuming the download if the server supports it - // http://stackoverflow.com/questions/41444297/ - // http://stackoverflow.com/questions/3428102/ + fn download(&self, out_dir: &Path) -> Result<(), DownloadError> { + fn download_to_file_and_check bool>(response: &mut Response, + out_file: &Path, + check: F) + -> Result<(), io::Error> { + println!("Output file: {}", out_file.display()); + if out_file.exists() { + println!("File is present already. Checking validity..."); + if check(out_file) { + println!("Valid file! Skipping!"); + return Ok(()); + } else { + println!("Corrupt file! Starting anew!"); } - let file = download_to_file(response, &out_file); - ::std::mem::drop(file); - assert!(check(out_file)); + // TODO?: We could try resuming the download if the server supports it + // http://stackoverflow.com/questions/41444297/ + // http://stackoverflow.com/questions/3428102/ + } + download_to_file(response, out_file)?; + assert!(check(out_file)); + Ok(()) } match *self { - Presentation::PublishToGo {ref download_url, ..} => { - let mut response = - try_to_get_valid_response(|client| { - client.get(download_url) - .header(construct_cookie(auth)) - }, - |resp| resp.headers().get::().is_some()); + Presentation::PublishToGo { ref download_url, .. } => { + let mut response = try_to_get_valid_response(|client| { + client + .get(download_url) + .header(construct_cookie()) + }, + |resp| { + resp.headers().get::().is_some() + }); let filename = { - let content_disposition = response.headers().get::().unwrap(); + let content_disposition = + response.headers().get::().unwrap(); assert_eq!(content_disposition.disposition, DispositionType::Attachment); let mut name = None; for param in &content_disposition.parameters { @@ -112,7 +128,7 @@ impl Presentation { DispositionParam::Filename(ref charset, _, ref vec) => { assert_eq!(&Charset::Ext("UTF-8".to_string()), charset); name = Some(String::from_utf8(vec.to_vec()) - .expect("Suggested filename isn't valid UTF-8!")); + .expect("Suggested filename isn't valid UTF-8!",)); } _ => continue, } @@ -123,19 +139,36 @@ impl Presentation { let mut path = PathBuf::from(out_dir); path.push(fix_filename(&filename)); - download_to_file_and_check(&mut response, &path, |path| zip_is_valid(path)); - }, - Presentation::VideoOnDemand {ref resource_id, ref query_string, ..} => { + download_to_file_and_check(&mut response, &path, |path| zip_is_valid(path)) + .map_err(|e| DownloadError::IoError(e)) + } + Presentation::VideoOnDemand { + ref resource_id, + ref query_string, + .. + } => { let videos = { let payload = format!("{{\"getPlayerOptionsRequest\":{{\ \"ResourceId\":\"{}\",\ \"QueryString\":\"{}\"\ - }}}}", resource_id, query_string); + }}}}", + resource_id, + query_string); - let url = "https://streams.tum.de/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions"; - let mut res = try_to_get_response(|client| client.post(url).header(construct_cookie(auth)).header(reqwest::header::ContentType::json()).body(&*payload)); + let url = "https://streams.tum.de/Mediasite/PlayerService/\ + PlayerService.svc/json/GetPlayerOptions"; + let mut res = try_to_get_response(|client| { + client + .post(url) + .header(construct_cookie()) + .header(reqwest::header::ContentType::json()) + .body(&*payload) + }); let json_text = read_response_body(&mut res); + if json_text.contains("You are not authorized to view the requested content") { + return Err(DownloadError::AuthorizationTimeout); + } let json = json::parse(&json_text).expect("Failed parsing the json!"); let mut vec = Vec::new(); @@ -148,12 +181,15 @@ impl Presentation { }; for (i, download_url) in videos.iter().enumerate() { let mut response = try_to_get_valid_response(|client| { - client.get(download_url) - .header(construct_cookie(auth)) - }, - |resp| resp.status() == &StatusCode::Ok); - // TODO: Support formats besides mp4; extract extension from url or somewehre else... - let filename = format!("{} - {}.mp4", fix_filename(presentation.name()), i + 1); + client.get(download_url).header(construct_cookie()) + }, + |resp| { + resp.status() == + &StatusCode::Ok + }); + // TODO: Support formats besides mp4 + // extract extension from url or somewehre else... + let filename = format!("{} - {}.mp4", fix_filename(self.name()), i + 1); let mut path = PathBuf::from(out_dir); path.push(fix_filename(&filename)); @@ -161,9 +197,11 @@ impl Presentation { let mut path = PathBuf::from(out_dir); path.push(fix_filename(&filename)); - download_to_file_and_check(&mut response, &path, |path| check_video(path)); + download_to_file_and_check(&mut response, &path, |path| check_video(path)) + .map_err(|e| DownloadError::IoError(e))?; } - }, + Ok(()) + } } } } @@ -191,7 +229,10 @@ impl<'a> From<&'a JsonValue> for Presentation { let index_start_resource = index - "cdf23465581b45aa92d588ebd59619d91d".len(); Presentation::VideoOnDemand { - name: json["Name"].clone().take_string().expect("json: Presentation didn't contain a \"Name\"!"), + name: json["Name"] + .clone() + .take_string() + .expect("json: Presentation didn't contain a \"Name\"!"), query_string: player_url[index..index_end_query].to_string(), resource_id: player_url[index_start_resource..index].to_string(), } @@ -204,25 +245,25 @@ fn main() { .author("Boris-Chengbiao Zhou ") .about("Downloads \'catalogs\' from the TUM's Mediasite lecture archive.") .arg(Arg::with_name("CATALOG_NAME") - .help("name of the catalog e.g. from the URL:\n\ + .help("name of the catalog e.g. from the URL:\n\ https://streams.tum.de/Mediasite/Catalog/catalogs/era-2016 -> era-2016\n\ special cases (WS16/17; login included): DS, EIDI, ERA") - .required(true) - .index(1)) + .required(true) + .index(1)) .arg(Arg::with_name("OUTPUT_DIRECTORY") - .help("where to output the downloaded files") - .required(true) - .index(2)) + .help("where to output the downloaded files") + .required(true) + .index(2)) .arg(Arg::with_name("username") - .short("u") - .help("username for login; can be omitted if the user from .env should be used") - .requires("password") - .takes_value(true)) + .short("u") + .help("username for login; can be omitted if the user from .env should be used",) + .requires("password") + .takes_value(true)) .arg(Arg::with_name("password") - .short("p") - .help("password for login; can be omitted if the user from .env should be used") - .requires("username") - .takes_value(true)) + .short("p") + .help("password for login; can be omitted if the user from .env should be used",) + .requires("username") + .takes_value(true)) .get_matches(); let catalog_def = match matches.value_of("CATALOG_NAME").unwrap() { @@ -256,8 +297,11 @@ fn main() { println!("Preparing to download catalog \"{}\"!", catalog_name); - let auth = get_auth(&username, &password); - download_catalog(catalog_name, out_dir, &auth); + USERNAME.lock().unwrap().push_str(&username); + PASSWORD.lock().unwrap().push_str(&password); + get_auth(); + + download_catalog(catalog_name, out_dir); } fn get_default_login() -> (String, String) { @@ -270,19 +314,23 @@ fn get_default_login() -> (String, String) { (username, password) } -fn get_auth(username: &str, password: &str) -> String { +fn get_auth() { println!("Logging in!"); + let username = USERNAME.lock().unwrap(); + let password = PASSWORD.lock().unwrap(); + let mut form_data = HashMap::new(); - form_data.insert("UserName", username); - form_data.insert("Password", password); + form_data.insert("UserName", &*username); + form_data.insert("Password", &*password); let res = try_to_get_valid_response(|client| { - client.post("https://streams.tum.de/Mediasite/Login") + client + .post("https://streams.tum.de/Mediasite/Login") .form(&form_data) }, |res| res.headers().get::().is_some()); - // FIXME: We're somehow only getting "Object moved" instead of the actual response + // FIXME: We're somehow only getting "302 Object moved" instead of the actual response // => We can't determine if the login was successful // (we still get a MediasiteAuth cookie that is useless) // let body = read_response_body(&mut res); @@ -295,30 +343,45 @@ fn get_auth(username: &str, password: &str) -> String { .expect("Failed to parse SetCookie"); assert_eq!(cookie.name(), "MediasiteAuth"); - cookie.value().to_string() + let mut auth = AUTH.lock().unwrap(); + auth.clear(); + auth.push_str(cookie.value()); } -fn download_catalog(catalog_name: &str, out_dir: &Path, auth: &str) { - let catalog_id = get_catalog_id(catalog_name, auth); - let json = get_json(&catalog_id, auth); +fn download_catalog(catalog_name: &str, out_dir: &Path) { + let catalog_id = get_catalog_id(catalog_name); + let json = get_json(&catalog_id); let presentations = json_to_presentations(&json); println!("Starting to download {} presentations!", presentations.len()); for (i, presentation) in presentations.iter().enumerate() { - println!("Downloading {}/{}: {}", + println!("-------------------------------------------------"); + println!("\nDownloading {}/{}: {}", i + 1, presentations.len(), presentation.name()); - presentation.download(presentation, out_dir, auth); + for _ in 0..MAX_RETRIES { + match presentation.download(out_dir) { + Ok(()) => break, + Err(DownloadError::IoError(e)) => { + println!("Error during download: {:?}", e); + println!("Retrying!"); + } + Err(DownloadError::AuthorizationTimeout) => { + println!("Authorization is not valid anymore. Refreshing!"); + get_auth(); + } + } + } } } -fn get_catalog_id(name: &str, auth: &str) -> String { +fn get_catalog_id(name: &str) -> String { println!("Fetching catalog id!"); let url = format!("https://streams.tum.de/Mediasite/Catalog/catalogs/{}", name); - let mut res = try_to_get_response(|client| client.get(&url).header(construct_cookie(auth))); + let mut res = try_to_get_response(|client| client.get(&url).header(construct_cookie())); let body = read_response_body(&mut res); let prefix = "CatalogId: '"; @@ -332,51 +395,55 @@ fn get_catalog_id(name: &str, auth: &str) -> String { body[(idx + pre_len)..(idx + pre_len + len)].to_string() } -fn download_to_file(response: &mut Response, path: &Path) -> File { - let mut file = File::create(&path).expect("Failed to create file!"); +fn download_to_file(response: &mut Response, path: &Path) -> Result<(), io::Error> { + let mut file = File::create(&path).expect("Failed to create file!"); - let mut reader = BufReader::new(response); - let mut buf = [0u8; 8 * 1024]; + let mut reader = BufReader::new(response); + let mut buf = [0u8; 8 * 1024]; - let update_duration_millis = 1000; - let mut last = Instant::now(); - let mut done = 0; + let update_duration_millis = 1000; + let mut last = Instant::now(); + let mut done = 0; - loop { - let len = match reader.read(&mut buf) { - Ok(0) => break, - Ok(len) => len, - Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, - Err(e) => panic!(e), - }; + loop { + let len = match reader.read(&mut buf) { + Ok(0) => break, + Ok(len) => len, + Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, + Err(e) => return Err(e), + }; - file.write_all(&buf[0..len]).expect("Failed writing into the file!"); - done += len; + file.write_all(&buf[0..len]) + .expect("Failed writing into the file!"); + done += len; - let now = Instant::now(); - if now - last > Duration::from_millis(update_duration_millis) { - print!("{:.*} kB/s \r", - 2, - done as f64 * 1000.0 / update_duration_millis as f64 / 1024.0); - std::io::stdout().flush().expect("Failed flushing the terminal!"); - last = now; - done = 0; - } + let now = Instant::now(); + if now - last > Duration::from_millis(update_duration_millis) { + print!("{:.*} kB/s \r", + 2, + done as f64 * 1000.0 / update_duration_millis as f64 / 1024.0); + std::io::stdout() + .flush() + .expect("Failed flushing the terminal!"); + last = now; + done = 0; } + } - file + Ok(()) } fn fix_filename(string: &str) -> String { - string.replace('/', "_") - .replace('\\', "_") - .replace(':', " -") - .replace('*', "%2A") - .replace('?', "%3F") - .replace('"', "'") - .replace('<', "'") - .replace('>', "'") - .replace('|', "_") + string + .replace('/', "_") + .replace('\\', "_") + .replace(':', " -") + .replace('*', "%2A") + .replace('?', "%3F") + .replace('"', "'") + .replace('<', "'") + .replace('>', "'") + .replace('|', "_") } fn json_to_presentations(json_str: &str) -> Vec { @@ -391,7 +458,7 @@ fn json_to_presentations(json_str: &str) -> Vec { vec } -fn get_json(catalog_id: &str, auth: &str) -> String { +fn get_json(catalog_id: &str) -> String { println!("Fetching catalog!"); let mut data = HashMap::new(); @@ -400,15 +467,16 @@ fn get_json(catalog_id: &str, auth: &str) -> String { data.insert("ItemsPerPage", "500"); let mut res = try_to_get_response(|client| { - client.post("https://streams.tum.de/Mediasite/Catalog/Data/GetPresentationsForFolder") - .header(construct_cookie(auth)) + client + .post("https://streams.tum.de/Mediasite/Catalog/Data/GetPresentationsForFolder") + .header(construct_cookie()) .json(&data) }); read_response_body(&mut res) } -fn construct_cookie(auth: &str) -> Cookie { - Cookie(vec![format!("MediasiteAuth={}", auth)]) +fn construct_cookie() -> Cookie { + Cookie(vec![format!("MediasiteAuth={}", *AUTH.lock().unwrap())]) } fn zip_is_valid(path: &Path) -> bool { @@ -430,12 +498,11 @@ fn check_video(path: &Path) -> bool { .output(); match command { - Ok(output) => { - output.stderr.is_empty() - } + Ok(output) => output.stderr.is_empty(), Err(_) => { println!("Failed to check {} with ffmpeg. Perhaps ffmpeg isn't in your path. \ - Skipping check...", path.display()); + Skipping check...", + path.display()); true } } @@ -451,8 +518,7 @@ fn try_to_get_valid_response(f1: F1, f2: F2) -> Response where F1: Fn(&Client) -> RequestBuilder, F2: Fn(&Response) -> bool { - let mut retries = 0; - while retries <= MAX_RETRIES { + for retries in 0..MAX_RETRIES { if retries > 0 { println!("Retrying request!"); } @@ -462,13 +528,14 @@ fn try_to_get_valid_response(f1: F1, f2: F2) -> Response return response; } } - retries += 1; } panic!("Reached maximum amount of retries!") } fn read_response_body(response: &mut Response) -> String { let mut string = String::new(); - response.read_to_string(&mut string).expect("Failed to read body"); + response + .read_to_string(&mut string) + .expect("Failed to read body"); string }