From 7cdca0b7d75172d8f900e8e755ff07ed8a72966a Mon Sep 17 00:00:00 2001 From: Jokler Date: Thu, 1 Mar 2018 17:27:49 +0100 Subject: Simplify the download function --- src/error.rs | 8 ++++---- src/utils.rs | 29 ++++------------------------- 2 files changed, 8 insertions(+), 29 deletions(-) diff --git a/src/error.rs b/src/error.rs index 2ceb4cc..674faaa 100644 --- a/src/error.rs +++ b/src/error.rs @@ -30,9 +30,9 @@ pub enum FrippyError { #[fail(display = "An I/O error occured")] Io(#[cause] IoError), - /// A UTF8 error - #[fail(display = "A UTF8 error occured")] - Utf8(#[cause] Utf8Error), + /// A decoding error + #[fail(display = "Failed to decode bytes")] + Decoding(#[cause] Utf8Error), /// An r2d2 error #[cfg(feature = "mysql")] @@ -100,7 +100,7 @@ impl From for FrippyError { impl From for FrippyError { fn from(e: Utf8Error) -> FrippyError { - FrippyError::Utf8(e) + FrippyError::Decoding(e) } } diff --git a/src/utils.rs b/src/utils.rs index 68ad4d8..1f3a117 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -9,43 +9,21 @@ use error::FrippyError; pub fn download(max_kib: usize, url: &str) -> Result { let mut response = Client::new().get(url).header(Connection::close()).send()?; - let mut body = String::new(); - // 100 kibibyte buffer let mut buf = [0; 100 * 1024]; let mut written = 0; - let mut vec = Vec::new(); - let mut end_of_valid = None; + let mut bytes = Vec::new(); // Read until we reach EOF or max_kib KiB loop { - if let Some(eov) = end_of_valid { - vec = vec[..eov].to_vec(); - } - let len = match response.read(&mut buf) { Ok(0) => break, Ok(len) => len, Err(ref e) if e.kind() == io::ErrorKind::Interrupted => continue, Err(e) => Err(e)?, }; - vec.extend_from_slice(&buf); - - end_of_valid = None; - let body_slice = match str::from_utf8(&vec[..len]) { - Ok(slice) => slice, - Err(e) => { - let valid = e.valid_up_to(); - if valid == 0 { - Err(e)?; - } - end_of_valid = Some(valid); - - str::from_utf8(&buf[..valid])? - } - }; - body.push_str(body_slice); + bytes.extend_from_slice(&buf); written += len; // Check if the file is too large to download @@ -53,6 +31,7 @@ pub fn download(max_kib: usize, url: &str) -> Result { Err(FrippyError::DownloadLimit { limit: max_kib })?; } } + let body = str::from_utf8(&bytes)?; - Ok(body) + Ok(body.to_string()) } -- cgit v1.2.3-70-g09d2 From d8406b107c651321c9e166abc36fc66730d965a1 Mon Sep 17 00:00:00 2001 From: Jokler Date: Thu, 1 Mar 2018 18:34:03 +0100 Subject: Use lossy UTF8 conversion and add a log_error function --- src/error.rs | 11 ----------- src/main.rs | 9 +-------- src/plugins/factoids/mod.rs | 2 +- src/plugins/factoids/utils.rs | 2 +- src/plugins/url.rs | 7 ++----- src/utils.rs | 26 +++++++++++++++++++++----- 6 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/error.rs b/src/error.rs index 674faaa..fa232be 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,7 +1,6 @@ //! Errors for `frippy` crate using `failure`. use std::io::Error as IoError; -use std::str::Utf8Error; use irc::error::IrcError; use reqwest::Error as ReqwestError; #[cfg(feature = "mysql")] @@ -30,10 +29,6 @@ pub enum FrippyError { #[fail(display = "An I/O error occured")] Io(#[cause] IoError), - /// A decoding error - #[fail(display = "Failed to decode bytes")] - Decoding(#[cause] Utf8Error), - /// An r2d2 error #[cfg(feature = "mysql")] #[fail(display = "An r2d2 error occured")] @@ -98,12 +93,6 @@ impl From for FrippyError { } } -impl From for FrippyError { - fn from(e: Utf8Error) -> FrippyError { - FrippyError::Decoding(e) - } -} - #[cfg(feature = "mysql")] impl From for FrippyError { fn from(e: R2d2Error) -> FrippyError { diff --git a/src/main.rs b/src/main.rs index 461387d..3432e3e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -5,7 +5,6 @@ extern crate frippy; extern crate glob; extern crate irc; extern crate time; -extern crate failure; #[cfg(feature = "mysql")] extern crate diesel; @@ -27,7 +26,6 @@ use log::{Level, LevelFilter, Metadata, Record}; use irc::client::reactor::IrcReactor; use glob::glob; -use failure::Fail; use frippy::plugins; use frippy::Config; @@ -72,12 +70,7 @@ static LOGGER: Logger = Logger; fn main() { // Print any errors that caused frippy to shut down if let Err(e) = run() { - let mut causes = e.causes(); - - error!("{}", causes.next().unwrap()); - for cause in causes { - error!("caused by: {}", cause); - } + frippy::utils::log_error(e); }; } diff --git a/src/plugins/factoids/mod.rs b/src/plugins/factoids/mod.rs index 3f89943..806bb7e 100644 --- a/src/plugins/factoids/mod.rs +++ b/src/plugins/factoids/mod.rs @@ -83,7 +83,7 @@ impl Factoids { let name = command.tokens.remove(0); let url = &command.tokens[0]; - let content = ::utils::download(1024, url)?; + let content = ::utils::download(url, Some(1024))?; Ok(self.create_factoid(&name, &content, &command.source)?) } diff --git a/src/plugins/factoids/utils.rs b/src/plugins/factoids/utils.rs index 036dcc6..009b46b 100644 --- a/src/plugins/factoids/utils.rs +++ b/src/plugins/factoids/utils.rs @@ -9,7 +9,7 @@ use super::rlua::prelude::*; use self::LuaError::RuntimeError; pub fn download(_: &Lua, url: String) -> Result { - match utils::download(1024, &url) { + match utils::download(&url, Some(1024)) { Ok(v) => Ok(v), Err(e) => Err(RuntimeError(format!("Failed to download {} - {}", url, e.to_string()))), } diff --git a/src/plugins/url.rs b/src/plugins/url.rs index af6f36f..6f00466 100644 --- a/src/plugins/url.rs +++ b/src/plugins/url.rs @@ -50,7 +50,7 @@ impl Url { fn url(&self, text: &str) -> Result { let url = self.grep_url(text).ok_or(UrlError::MissingUrl)?; - let body = utils::download(self.max_kib, &url)?; + let body = utils::download(&url, Some(self.max_kib))?; Ok(self.get_title(&body).ok_or(UrlError::MissingTitle)?) } @@ -72,10 +72,7 @@ impl Plugin for Url { match message.command { Command::PRIVMSG(_, ref content) => match self.url(content) { Ok(title) => client.send_privmsg(message.response_target().unwrap(), &title), - Err(e) => { - error!("Url plugin error: {}", e); - Ok(()) - } + Err(e) => Ok(utils::log_error(e)), }, _ => Ok(()), } diff --git a/src/utils.rs b/src/utils.rs index 1f3a117..cf91b37 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -4,9 +4,14 @@ use std::io::{self, Read}; use reqwest::Client; use reqwest::header::Connection; +use failure::Fail; use error::FrippyError; -pub fn download(max_kib: usize, url: &str) -> Result { +/// Downloads the file and converts it to a String. +/// Any invalid bytes are converted to a replacement character. +/// +/// The error indicated either a failed download or that the DownloadLimit was reached +pub fn download(url: &str, max_kib: Option) -> Result { let mut response = Client::new().get(url).header(Connection::close()).send()?; // 100 kibibyte buffer @@ -27,11 +32,22 @@ pub fn download(max_kib: usize, url: &str) -> Result { written += len; // Check if the file is too large to download - if written > max_kib * 1024 { - Err(FrippyError::DownloadLimit { limit: max_kib })?; + if let Some(max_kib) = max_kib { + if written > max_kib * 1024 { + Err(FrippyError::DownloadLimit { limit: max_kib })?; + } } } - let body = str::from_utf8(&bytes)?; - Ok(body.to_string()) + Ok(String::from_utf8_lossy(&bytes).into_owned()) +} + + +pub fn log_error(e: FrippyError) { + let mut causes = e.causes(); + + error!("{}", causes.next().unwrap()); + for cause in causes { + error!("caused by: {}", cause); + } } -- cgit v1.2.3-70-g09d2