bobo1239/TumMediasiteDownloader
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Make AUTH a global variable and refresh authorization when our previous one timed out; Re-enable ffmpeg check

Browse Source
This commit is contained in:
Boris-Chengbiao Zhou
2017-05-15 12:42:15 +02:00
parent 3c3bf8554f
commit 51e4f591d6
2 changed files with 204 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

317
src/main.rs
View File

@@ -14,6 +14,7 @@ use std::fs::File;
use std::io::{self, Read, BufReader, Write};
use std::path::{Path, PathBuf};
use std::time::{Instant, Duration};
use std::sync::Mutex;
use clap::{App, Arg};
use json::JsonValue;
@@ -27,6 +28,8 @@ use catalogs::*;
// TODO: Old "Publish To Go" packages get sweeped from the server and you have to request a new one;
// Implement this
// TODO: sometimes you need to access the video listing at least once using moodle; emulate this
// TODO: Add aliases with semester postfix so other can contribute aliases; move to separate file
// split into files in general?
mod catalogs {
type Login = Option<(&'static str, &'static str)>;
@@ -51,60 +54,73 @@ lazy_static! {
client.redirect(RedirectPolicy::none());
client
};
static ref USERNAME: Mutex<String> = Mutex::new(String::new());
static ref PASSWORD: Mutex<String> = Mutex::new(String::new());
static ref AUTH: Mutex<String> = Mutex::new(String::new());
}
#[derive(Debug)]
enum DownloadError {
IoError(io::Error),
AuthorizationTimeout,
}
#[derive(Debug)]
enum Presentation {
PublishToGo {
name: String,
download_url: String,
},
PublishToGo { name: String, download_url: String },
VideoOnDemand {
name: String,
resource_id: String,
query_string: String,
}
},
}
impl Presentation {
fn name(&self) -> &str {
match *self {
Presentation::PublishToGo {ref name, ..} |
Presentation::VideoOnDemand {ref name, ..} => {name},
Presentation::PublishToGo { ref name, .. } |
Presentation::VideoOnDemand { ref name, .. } => name,
}
}
fn download(&self, presentation: &Presentation, out_dir: &Path, auth: &str) {
fn download_to_file_and_check<F: Fn(&Path) -> bool>(response: &mut Response, out_file: &Path, check: F) {
println!("Output file: {}", out_file.display());
if out_file.exists() {
if check(out_file) {
println!("Already present and valid! Skipping!");
return;
} else {
println!("Found corrupt file! Starting anew!");
}
// TODO?: We could try resuming the download if the server supports it
// http://stackoverflow.com/questions/41444297/
// http://stackoverflow.com/questions/3428102/
fn download(&self, out_dir: &Path) -> Result<(), DownloadError> {
fn download_to_file_and_check<F: Fn(&Path) -> bool>(response: &mut Response,
out_file: &Path,
check: F)
-> Result<(), io::Error> {
println!("Output file: {}", out_file.display());
if out_file.exists() {
println!("File is present already. Checking validity...");
if check(out_file) {
println!("Valid file! Skipping!");
return Ok(());
} else {
println!("Corrupt file! Starting anew!");
}
let file = download_to_file(response, &out_file);
::std::mem::drop(file);
assert!(check(out_file));
// TODO?: We could try resuming the download if the server supports it
// http://stackoverflow.com/questions/41444297/
// http://stackoverflow.com/questions/3428102/
}
download_to_file(response, out_file)?;
assert!(check(out_file));
Ok(())
}
match *self {
Presentation::PublishToGo {ref download_url, ..} => {
let mut response =
try_to_get_valid_response(|client| {
client.get(download_url)
.header(construct_cookie(auth))
},
|resp| resp.headers().get::<ContentDisposition>().is_some());
Presentation::PublishToGo { ref download_url, .. } => {
let mut response = try_to_get_valid_response(|client| {
client
.get(download_url)
.header(construct_cookie())
},
|resp| {
resp.headers().get::<ContentDisposition>().is_some()
});
let filename = {
let content_disposition = response.headers().get::<ContentDisposition>().unwrap();
let content_disposition =
response.headers().get::<ContentDisposition>().unwrap();
assert_eq!(content_disposition.disposition, DispositionType::Attachment);
let mut name = None;
for param in &content_disposition.parameters {
@@ -112,7 +128,7 @@ impl Presentation {
DispositionParam::Filename(ref charset, _, ref vec) => {
assert_eq!(&Charset::Ext("UTF-8".to_string()), charset);
name = Some(String::from_utf8(vec.to_vec())
.expect("Suggested filename isn't valid UTF-8!"));
.expect("Suggested filename isn't valid UTF-8!",));
}
_ => continue,
}
@@ -123,19 +139,36 @@ impl Presentation {
let mut path = PathBuf::from(out_dir);
path.push(fix_filename(&filename));
download_to_file_and_check(&mut response, &path, |path| zip_is_valid(path));
},
Presentation::VideoOnDemand {ref resource_id, ref query_string, ..} => {
download_to_file_and_check(&mut response, &path, |path| zip_is_valid(path))
.map_err(|e| DownloadError::IoError(e))
}
Presentation::VideoOnDemand {
ref resource_id,
ref query_string,
..
} => {
let videos = {
let payload = format!("{{\"getPlayerOptionsRequest\":{{\
\"ResourceId\":\"{}\",\
\"QueryString\":\"{}\"\
}}}}", resource_id, query_string);
}}}}",
resource_id,
query_string);
let url = "https://streams.tum.de/Mediasite/PlayerService/PlayerService.svc/json/GetPlayerOptions";
let mut res = try_to_get_response(|client| client.post(url).header(construct_cookie(auth)).header(reqwest::header::ContentType::json()).body(&*payload));
let url = "https://streams.tum.de/Mediasite/PlayerService/\
PlayerService.svc/json/GetPlayerOptions";
let mut res = try_to_get_response(|client| {
client
.post(url)
.header(construct_cookie())
.header(reqwest::header::ContentType::json())
.body(&*payload)
});
let json_text = read_response_body(&mut res);
if json_text.contains("You are not authorized to view the requested content") {
return Err(DownloadError::AuthorizationTimeout);
}
let json = json::parse(&json_text).expect("Failed parsing the json!");
let mut vec = Vec::new();
@@ -148,12 +181,15 @@ impl Presentation {
};
for (i, download_url) in videos.iter().enumerate() {
let mut response = try_to_get_valid_response(|client| {
client.get(download_url)
.header(construct_cookie(auth))
},
|resp| resp.status() == &StatusCode::Ok);
// TODO: Support formats besides mp4; extract extension from url or somewehre else...
let filename = format!("{} - {}.mp4", fix_filename(presentation.name()), i + 1);
client.get(download_url).header(construct_cookie())
},
|resp| {
resp.status() ==
&StatusCode::Ok
});
// TODO: Support formats besides mp4
// extract extension from url or somewehre else...
let filename = format!("{} - {}.mp4", fix_filename(self.name()), i + 1);
let mut path = PathBuf::from(out_dir);
path.push(fix_filename(&filename));
@@ -161,9 +197,11 @@ impl Presentation {
let mut path = PathBuf::from(out_dir);
path.push(fix_filename(&filename));
download_to_file_and_check(&mut response, &path, |path| check_video(path));
download_to_file_and_check(&mut response, &path, |path| check_video(path))
.map_err(|e| DownloadError::IoError(e))?;
}
},
Ok(())
}
}
}
}
@@ -191,7 +229,10 @@ impl<'a> From<&'a JsonValue> for Presentation {
let index_start_resource = index - "cdf23465581b45aa92d588ebd59619d91d".len();
Presentation::VideoOnDemand {
name: json["Name"].clone().take_string().expect("json: Presentation didn't contain a \"Name\"!"),
name: json["Name"]
.clone()
.take_string()
.expect("json: Presentation didn't contain a \"Name\"!"),
query_string: player_url[index..index_end_query].to_string(),
resource_id: player_url[index_start_resource..index].to_string(),
}
@@ -204,25 +245,25 @@ fn main() {
.author("Boris-Chengbiao Zhou <bobo1239@web.de>")
.about("Downloads \'catalogs\' from the TUM's Mediasite lecture archive.")
.arg(Arg::with_name("CATALOG_NAME")
.help("name of the catalog e.g. from the URL:\n\
.help("name of the catalog e.g. from the URL:\n\
https://streams.tum.de/Mediasite/Catalog/catalogs/era-2016 -> era-2016\n\
special cases (WS16/17; login included): DS, EIDI, ERA")
.required(true)
.index(1))
.required(true)
.index(1))
.arg(Arg::with_name("OUTPUT_DIRECTORY")
.help("where to output the downloaded files")
.required(true)
.index(2))
.help("where to output the downloaded files")
.required(true)
.index(2))
.arg(Arg::with_name("username")
.short("u")
.help("username for login; can be omitted if the user from .env should be used")
.requires("password")
.takes_value(true))
.short("u")
.help("username for login; can be omitted if the user from .env should be used",)
.requires("password")
.takes_value(true))
.arg(Arg::with_name("password")
.short("p")
.help("password for login; can be omitted if the user from .env should be used")
.requires("username")
.takes_value(true))
.short("p")
.help("password for login; can be omitted if the user from .env should be used",)
.requires("username")
.takes_value(true))
.get_matches();
let catalog_def = match matches.value_of("CATALOG_NAME").unwrap() {
@@ -256,8 +297,11 @@ fn main() {
println!("Preparing to download catalog \"{}\"!", catalog_name);
let auth = get_auth(&username, &password);
download_catalog(catalog_name, out_dir, &auth);
USERNAME.lock().unwrap().push_str(&username);
PASSWORD.lock().unwrap().push_str(&password);
get_auth();
download_catalog(catalog_name, out_dir);
}
fn get_default_login() -> (String, String) {
@@ -270,19 +314,23 @@ fn get_default_login() -> (String, String) {
(username, password)
}
fn get_auth(username: &str, password: &str) -> String {
fn get_auth() {
println!("Logging in!");
let username = USERNAME.lock().unwrap();
let password = PASSWORD.lock().unwrap();
let mut form_data = HashMap::new();
form_data.insert("UserName", username);
form_data.insert("Password", password);
form_data.insert("UserName", &*username);
form_data.insert("Password", &*password);
let res = try_to_get_valid_response(|client| {
client.post("https://streams.tum.de/Mediasite/Login")
client
.post("https://streams.tum.de/Mediasite/Login")
.form(&form_data)
},
|res| res.headers().get::<SetCookie>().is_some());
// FIXME: We're somehow only getting "Object moved" instead of the actual response
// FIXME: We're somehow only getting "302 Object moved" instead of the actual response
// => We can't determine if the login was successful
// (we still get a MediasiteAuth cookie that is useless)
// let body = read_response_body(&mut res);
@@ -295,30 +343,45 @@ fn get_auth(username: &str, password: &str) -> String {
.expect("Failed to parse SetCookie");
assert_eq!(cookie.name(), "MediasiteAuth");
cookie.value().to_string()
let mut auth = AUTH.lock().unwrap();
auth.clear();
auth.push_str(cookie.value());
}
fn download_catalog(catalog_name: &str, out_dir: &Path, auth: &str) {
let catalog_id = get_catalog_id(catalog_name, auth);
let json = get_json(&catalog_id, auth);
fn download_catalog(catalog_name: &str, out_dir: &Path) {
let catalog_id = get_catalog_id(catalog_name);
let json = get_json(&catalog_id);
let presentations = json_to_presentations(&json);
println!("Starting to download {} presentations!",
presentations.len());
for (i, presentation) in presentations.iter().enumerate() {
println!("Downloading {}/{}: {}",
println!("-------------------------------------------------");
println!("\nDownloading {}/{}: {}",
i + 1,
presentations.len(),
presentation.name());
presentation.download(presentation, out_dir, auth);
for _ in 0..MAX_RETRIES {
match presentation.download(out_dir) {
Ok(()) => break,
Err(DownloadError::IoError(e)) => {
println!("Error during download: {:?}", e);
println!("Retrying!");
}
Err(DownloadError::AuthorizationTimeout) => {
println!("Authorization is not valid anymore. Refreshing!");
get_auth();
}
}
}
}
}
fn get_catalog_id(name: &str, auth: &str) -> String {
fn get_catalog_id(name: &str) -> String {
println!("Fetching catalog id!");
let url = format!("https://streams.tum.de/Mediasite/Catalog/catalogs/{}", name);
let mut res = try_to_get_response(|client| client.get(&url).header(construct_cookie(auth)));
let mut res = try_to_get_response(|client| client.get(&url).header(construct_cookie()));
let body = read_response_body(&mut res);
let prefix = "CatalogId: '";
@@ -332,51 +395,55 @@ fn get_catalog_id(name: &str, auth: &str) -> String {
body[(idx + pre_len)..(idx + pre_len + len)].to_string()
}
fn download_to_file(response: &mut Response, path: &Path) -> File {
let mut file = File::create(&path).expect("Failed to create file!");
fn download_to_file(response: &mut Response, path: &Path) -> Result<(), io::Error> {
let mut file = File::create(&path).expect("Failed to create file!");
let mut reader = BufReader::new(response);
let mut buf = [0u8; 8 * 1024];
let mut reader = BufReader::new(response);
let mut buf = [0u8; 8 * 1024];
let update_duration_millis = 1000;
let mut last = Instant::now();
let mut done = 0;
let update_duration_millis = 1000;
let mut last = Instant::now();
let mut done = 0;
loop {
let len = match reader.read(&mut buf) {
Ok(0) => break,
Ok(len) => len,
Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => panic!(e),
};
loop {
let len = match reader.read(&mut buf) {
Ok(0) => break,
Ok(len) => len,
Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue,
Err(e) => return Err(e),
};
file.write_all(&buf[0..len]).expect("Failed writing into the file!");
done += len;
file.write_all(&buf[0..len])
.expect("Failed writing into the file!");
done += len;
let now = Instant::now();
if now - last > Duration::from_millis(update_duration_millis) {
print!("{:.*} kB/s \r",
2,
done as f64 * 1000.0 / update_duration_millis as f64 / 1024.0);
std::io::stdout().flush().expect("Failed flushing the terminal!");
last = now;
done = 0;
}
let now = Instant::now();
if now - last > Duration::from_millis(update_duration_millis) {
print!("{:.*} kB/s \r",
2,
done as f64 * 1000.0 / update_duration_millis as f64 / 1024.0);
std::io::stdout()
.flush()
.expect("Failed flushing the terminal!");
last = now;
done = 0;
}
}
file
Ok(())
}
fn fix_filename(string: &str) -> String {
string.replace('/', "_")
.replace('\\', "_")
.replace(':', " -")
.replace('*', "%2A")
.replace('?', "%3F")
.replace('"', "'")
.replace('<', "'")
.replace('>', "'")
.replace('|', "_")
string
.replace('/', "_")
.replace('\\', "_")
.replace(':', " -")
.replace('*', "%2A")
.replace('?', "%3F")
.replace('"', "'")
.replace('<', "'")
.replace('>', "'")
.replace('|', "_")
}
fn json_to_presentations(json_str: &str) -> Vec<Presentation> {
@@ -391,7 +458,7 @@ fn json_to_presentations(json_str: &str) -> Vec<Presentation> {
vec
}
fn get_json(catalog_id: &str, auth: &str) -> String {
fn get_json(catalog_id: &str) -> String {
println!("Fetching catalog!");
let mut data = HashMap::new();
@@ -400,15 +467,16 @@ fn get_json(catalog_id: &str, auth: &str) -> String {
data.insert("ItemsPerPage", "500");
let mut res = try_to_get_response(|client| {
client.post("https://streams.tum.de/Mediasite/Catalog/Data/GetPresentationsForFolder")
.header(construct_cookie(auth))
client
.post("https://streams.tum.de/Mediasite/Catalog/Data/GetPresentationsForFolder")
.header(construct_cookie())
.json(&data)
});
read_response_body(&mut res)
}
fn construct_cookie(auth: &str) -> Cookie {
Cookie(vec![format!("MediasiteAuth={}", auth)])
fn construct_cookie() -> Cookie {
Cookie(vec![format!("MediasiteAuth={}", *AUTH.lock().unwrap())])
}
fn zip_is_valid(path: &Path) -> bool {
@@ -430,12 +498,11 @@ fn check_video(path: &Path) -> bool {
.output();
match command {
Ok(output) => {
output.stderr.is_empty()
}
Ok(output) => output.stderr.is_empty(),
Err(_) => {
println!("Failed to check {} with ffmpeg. Perhaps ffmpeg isn't in your path. \
Skipping check...", path.display());
Skipping check...",
path.display());
true
}
}
@@ -451,8 +518,7 @@ fn try_to_get_valid_response<F1, F2>(f1: F1, f2: F2) -> Response
where F1: Fn(&Client) -> RequestBuilder,
F2: Fn(&Response) -> bool
{
let mut retries = 0;
while retries <= MAX_RETRIES {
for retries in 0..MAX_RETRIES {
if retries > 0 {
println!("Retrying request!");
}
@@ -462,13 +528,14 @@ fn try_to_get_valid_response<F1, F2>(f1: F1, f2: F2) -> Response
return response;
}
}
retries += 1;
}
panic!("Reached maximum amount of retries!")
}
fn read_response_body(response: &mut Response) -> String {
let mut string = String::new();
response.read_to_string(&mut string).expect("Failed to read body");
response
.read_to_string(&mut string)
.expect("Failed to read body");
string
}