extern crate clap; extern crate cookie; extern crate dotenv; extern crate json; #[macro_use] extern crate lazy_static; extern crate reqwest; extern crate zip; mod catalog; mod presentation; use std::env; use std::collections::HashMap; use std::io::{self, Read}; use std::path::Path; use std::sync::Mutex; use clap::{App, Arg}; use reqwest::{Client, ClientBuilder, Response, RedirectPolicy, RequestBuilder}; use reqwest::header::{Cookie, SetCookie}; use catalog::*; use presentation::Presentation; // TODO: Old "Publish To Go" packages get sweeped from the server and you have to request a new one; // Implement this // TODO: sometimes you need to access the video listing at least once using moodle; emulate this const MAX_RETRIES: u8 = 10; lazy_static! { static ref CLIENT: Client = ClientBuilder::new() .expect("Failed to create ClientBuilder!") // The login site redirects to itself if no redirect parameter is given .redirect(RedirectPolicy::none()) .build() .expect("Failed to create Client!"); static ref USERNAME: Mutex = Mutex::new(String::new()); static ref PASSWORD: Mutex = Mutex::new(String::new()); static ref AUTH: Mutex = Mutex::new(String::new()); } #[derive(Debug)] enum DownloadError { IoError(io::Error), AuthorizationTimeout, } fn main() { let matches = App::new("TumMediasiteDownloader") .author("Boris-Chengbiao Zhou ") .about( "Downloads \'catalogs\' from the TUM's Mediasite lecture archive.", ) .arg( Arg::with_name("CATALOG_NAME") .help( "name of the catalog e.g. from the URL:\n\ https://streams.tum.de/Mediasite/Catalog/catalogs/era-2016 -> era-2016\n\ special cases (WS16/17; login included): DS, EIDI, ERA, ...", ) .required(true) .index(1), ) .arg( Arg::with_name("OUTPUT_DIRECTORY") .help("where to output the downloaded files") .required(true) .index(2), ) .arg( Arg::with_name("username") .short("u") .help( "username for login; can be omitted if the user from .env should be used", ) .requires("password") .takes_value(true), ) .arg( Arg::with_name("password") .short("p") .help( "password for login; can be omitted if the user from .env should be used", ) .requires("username") .takes_value(true), ) .get_matches(); let catalog_name = matches.value_of("CATALOG_NAME").unwrap(); let login = if let Some(username) = matches.value_of("username") { let password = matches.value_of("password").unwrap(); Some((username, password)) } else { None }; let catalog_def = resolve_alias(catalog_name).unwrap_or((catalog_name, login)); let out_dir = Path::new(matches.value_of("OUTPUT_DIRECTORY").unwrap()); if out_dir.exists() { assert!(out_dir.is_dir()); } else { ::std::fs::create_dir_all(out_dir).expect("Failed to create output directory!"); } let (catalog_name, (username, password)) = if let Some((user, pass)) = catalog_def.1 { (catalog_def.0, (user.to_string(), pass.to_string())) } else { (catalog_def.0, get_default_login()) }; println!("Preparing to download catalog \"{}\"!", catalog_name); USERNAME.lock().unwrap().push_str(&username); PASSWORD.lock().unwrap().push_str(&password); get_auth(); download_catalog(catalog_name, out_dir); } fn get_default_login() -> (String, String) { if dotenv::dotenv().is_err() { println!("No .env found!"); } let username = env::var("TUM_USERNAME").expect("Missing TUM_USERNAME environment variable!"); let password = env::var("TUM_PASSWORD").expect("Missing TUM_PASSWORD environment variable!"); (username, password) } fn get_auth() { println!("Logging in!"); let username = USERNAME.lock().unwrap(); let password = PASSWORD.lock().unwrap(); let mut form_data = HashMap::new(); form_data.insert("UserName", &*username); form_data.insert("Password", &*password); let res = try_to_get_valid_response( |client| { let mut request_builder = client.post("https://streams.tum.de/Mediasite/Login").unwrap(); request_builder.form( &form_data, ).expect("Failed to serialize form_data!"); request_builder }, |res| res.headers().get::().is_some(), ).expect( "Didn't receive a valid response trying to login! Maybe wrong login data?", ); // FIXME: We're somehow only getting "302 Object moved" instead of the actual response // => We can't determine if the login was successful // (we still get a MediasiteAuth cookie that is useless) // let body = read_response_body(&mut res); // if body.contains("Unknown username or bad password.") { // panic!("Unknown username or bad password!"); // } let set_cookie: &SetCookie = res.headers().get().unwrap(); let cookie = cookie::Cookie::parse(set_cookie.0[0].to_string()).expect("Failed to parse SetCookie"); assert_eq!(cookie.name(), "MediasiteAuth"); let mut auth = AUTH.lock().unwrap(); auth.clear(); auth.push_str(cookie.value()); } fn download_catalog(catalog_name: &str, out_dir: &Path) { let catalog_id = get_catalog_id(catalog_name); let json = get_json(&catalog_id); let presentations = json_to_presentations(&json); println!( "Starting to download {} presentations!", presentations.len() ); for (i, presentation) in presentations.iter().enumerate() { println!( "\nDownloading {}/{}: {}", i + 1, presentations.len(), presentation.name() ); for _ in 0..MAX_RETRIES { match presentation.download(out_dir) { Ok(()) => break, Err(DownloadError::IoError(e)) => { println!("Error during download: {:?}", e); println!("Retrying!"); } Err(DownloadError::AuthorizationTimeout) => { println!("Authorization is not valid anymore. Refreshing!"); get_auth(); } } } } } fn get_catalog_id(name: &str) -> String { println!("Fetching catalog id!"); let url = format!("https://streams.tum.de/Mediasite/Catalog/catalogs/{}", name); let mut res = try_to_get_response(|client| { let mut request_builder = client.get(&url).unwrap(); request_builder.header(construct_cookie()); request_builder }); let body = read_response_body(&mut res); let prefix = "CatalogId: '"; let idx = body.find(prefix).expect( "Failed to find CatalogId on the catalog page! Perhaps you got the wrong catalog \ name or an invalid login? Maybe you need to open the page in a browser once...", ); let pre_len = prefix.len(); // Assuming all catalog ids follow this pattern! let len = "a6fca0c1-0be4-4e66-83b7-bcdc4eb5e95e".len(); body[(idx + pre_len)..(idx + pre_len + len)].to_string() } fn json_to_presentations(json_str: &str) -> Vec { let mut vec = Vec::new(); let mut json = json::parse(json_str).expect("Failed parsing the json!"); let mut count = 0; for presentation in json["PresentationDetailsList"].members_mut() { vec.push(Presentation::from(&*presentation)); count += 1; } assert_eq!(count, json["TotalItems"]); // Maybe there are multiple pages vec } fn get_json(catalog_id: &str) -> String { println!("Fetching catalog!"); let mut data = HashMap::new(); data.insert("CatalogId", catalog_id); data.insert("CurrentFolderId", catalog_id); data.insert("ItemsPerPage", "500"); let mut res = try_to_get_response(|client| { let mut request_builder = client .post( "https://streams.tum.de/Mediasite/Catalog/Data/GetPresentationsForFolder", ).unwrap(); request_builder.header(construct_cookie()) .json(&data).expect("Failed to serialize json!"); request_builder }); read_response_body(&mut res) } fn construct_cookie() -> Cookie { let mut cookie = Cookie::new(); cookie.append("MediasiteAuth", (*AUTH.lock().unwrap()).to_string()); cookie } fn try_to_get_response(f: F) -> Response where F: Fn(&Client) -> RequestBuilder, { try_to_get_valid_response(f, |_| true).unwrap() } fn try_to_get_valid_response(f1: F1, f2: F2) -> Result> where F1: Fn(&Client) -> RequestBuilder, F2: Fn(&Response) -> bool, { let mut last_response = None; for retries in 0..MAX_RETRIES { if retries > 0 { println!("Retrying request!"); } let response_res = f1(&*CLIENT).send(); if let Ok(response) = response_res { if f2(&response) { return Ok(response); } else { last_response = Some(Ok(response)); } } else { last_response = Some(response_res); } } println!( "Failed to get valid response! (tried {} times)", MAX_RETRIES ); Err(last_response.unwrap()) } fn read_response_body(response: &mut Response) -> String { let mut string = String::new(); response.read_to_string(&mut string).expect( "Failed to read body", ); string }