Fix UTF8 errors in the download function

1 files changed, 21 insertions, 6 deletions

diff --git a/src/utils.rs b/src/utils.rs
index 0b965c8..99b04c4 100644
--- a/src/utils.rs
+++ b/src/utils.rs
@@ -13,11 +13,18 @@ pub fn download(max_kib: usize, url: &str) -> Option<String> {
         Ok(mut response) => {
             let mut body = String::new();
 
-            // 500 kilobyte buffer
-            let mut buf = [0; 500 * 1000];
+            // 100 kibibyte buffer
+            let mut buf = [0; 100 * 1024];
             let mut written = 0;
+            let mut vec = Vec::new();
+            let mut end_of_valid = None;
+
             // Read until we reach EOF or max_kib KiB
             loop {
+                if let Some(eov) = end_of_valid {
+                    vec = vec[..eov].to_vec();
+                }
+
                 let len = match response.read(&mut buf) {
                     Ok(0) => break,
                     Ok(len) => len,
@@ -27,16 +34,24 @@ pub fn download(max_kib: usize, url: &str) -> Option<String> {
                         return None;
                     }
                 };
+                vec.extend_from_slice(&buf);
 
-                let slice = match str::from_utf8(&buf[..len]) {
+                end_of_valid = None;
+                let body_slice = match str::from_utf8(&vec[..len]) {
                     Ok(slice) => slice,
                     Err(e) => {
-                        debug!("Failed to read bytes from {:?} as UTF8: {}", url, e);
-                        return None;
+                        let valid = e.valid_up_to();
+                        if valid == 0 {
+                            error!("Failed to read bytes from {:?} as UTF8: {}", url, e);
+                            return None;
+                        }
+                        end_of_valid = Some(valid);
+
+                        str::from_utf8(&buf[..valid]).unwrap()
                     }
                 };
 
-                body.push_str(slice);
+                body.push_str(body_slice);
                 written += len;
 
                 // Check if the file is too large to download