diff --git a/README.md b/README.md index b5386cd39..d3d226559 100644 --- a/README.md +++ b/README.md @@ -241,7 +241,9 @@ Example filter: `((Group ~ "^DE.*") AND (NOT Title ~ ".*Shopping.*")) OR (Group If you use characters like `+ | [ ] ( )` in filters don't forget to escape them!! The regular expression syntax is similar to Perl-style regular expressions, -but lacks a few features like look around and backreferences. +but lacks a few features like look around and backreferences. +To test the regular expression i use [regex101.com](https://regex101.com/). +Don't forget to select `Rust` option which is under the `FLAVOR` section on the left. ### 1.5.2.6 `rename` Is a List of rename configurations. Each configuration has 3 top level entries. diff --git a/frontend/src/api/file-api-service.ts b/frontend/src/api/file-api-service.ts index 77c265e34..ac1f2c900 100644 --- a/frontend/src/api/file-api-service.ts +++ b/frontend/src/api/file-api-service.ts @@ -5,10 +5,11 @@ import {first} from "rxjs/operators"; //const FILE_API_PATH = 'file'; const FILE_DOWNLOAD_API_PATH = 'file/download'; +const FILE_DOWNLOAD_INFO_API_PATH =FILE_DOWNLOAD_API_PATH + '/info'; export default interface FileApiService extends ApiService { download(req: FileDownloadRequest): Observable; - getDownloadInfo(download_id: string): Observable; + getDownloadInfo(): Observable; } export class DefaultFileApiService extends DefaultApiService implements FileApiService { @@ -19,28 +20,25 @@ export class DefaultFileApiService extends DefaultApiService implements FileApiS return throwError(() => new Error('Invalid arguments')); } - getDownloadInfo(download_id: string): Observable { - if (download_id) { - return new Observable((observer) => { - const fetch_info = () => { - this.get(FILE_DOWNLOAD_API_PATH + '/' + download_id).pipe(first()).subscribe({ - next: (info: FileDownloadInfo) => { - if (info.finished) { - observer.next(info); - observer.complete(); - } else if (info.filesize != undefined) { - observer.next(info); - setTimeout(() => fetch_info(), 1000); - } else { - observer.error("unknown file download state"); - } - }, - error: (err) => observer.error(err) - }); - } - fetch_info(); - }); - } - return throwError(() => new Error('Invalid arguments')); + getDownloadInfo(): Observable { + return new Observable((observer) => { + const fetch_info = () => { + this.get(FILE_DOWNLOAD_INFO_API_PATH).pipe(first()).subscribe({ + next: (info: FileDownloadInfo) => { + if (info.finished) { + observer.next(info); + observer.complete(); + } else if (info.filesize != undefined) { + observer.next(info); + setTimeout(() => fetch_info(), 1000); + } else { + observer.error("unknown file download state"); + } + }, + error: (err) => observer.error(err), + }); + } + fetch_info(); + }); } } diff --git a/frontend/src/component/playlist-tree/playlist-tree.scss b/frontend/src/component/playlist-tree/playlist-tree.scss index 41a1b1b88..0effded5e 100644 --- a/frontend/src/component/playlist-tree/playlist-tree.scss +++ b/frontend/src/component/playlist-tree/playlist-tree.scss @@ -1,7 +1,7 @@ @use '../../scss/common'; .playlist-tree { - font-size: 12px; + font-size: 1rem; .tree-expander { margin-right: 6px; @@ -24,7 +24,7 @@ display: flex; flex-flow: row; align-items: center; - height: 1.3em; + height: 1.3rem; .tree-group-header-content { display: flex; @@ -49,7 +49,8 @@ flex-flow: row; align-items: center; margin-left: 50px; - height: 1.3em; + height: 1.3rem; + gap: 3px; .tree-channel-tools { display: flex; @@ -61,15 +62,15 @@ display: flex; align-items: center; justify-content: center; - height: 1em; - width: 1em; + height: 1.1rem; + width: 1.1rem; border-radius: var(--border-radius); background-color: var(--tool-button-background-color); color: var(--tool-button-color); svg { - width: 0.5em; - height: 0.5em; + width: 1rem; + height: 1rem; } &:hover { @@ -87,8 +88,8 @@ } .tree-channel-nr { - width: 20px; - font-size: 0.8em; + min-width: 2rem; + font-size: 1rem; margin-right: 8px; text-align: right; color: var(--tree-count-color); diff --git a/frontend/src/component/playlist-tree/playlist-tree.tsx b/frontend/src/component/playlist-tree/playlist-tree.tsx index 8df2342ee..1935fd0f1 100644 --- a/frontend/src/component/playlist-tree/playlist-tree.tsx +++ b/frontend/src/component/playlist-tree/playlist-tree.tsx @@ -12,8 +12,6 @@ import {FileDownloadInfo, FileDownloadResponse} from "../../model/file-download" const VALID_VIDEO_FILES = ['mkv', 'mp4', 'avi']; -type DownloadInfo = {filename: string, filesize: number}; - export type PlaylistTreeState = { [key: number]: boolean }; interface PlaylistTreeProps { @@ -32,7 +30,8 @@ export default function PlaylistTree(props: PlaylistTreeProps) { const {enqueueSnackbar/*, closeSnackbar*/} = useSnackbar(); const services = useServices(); const [videoExtensions, setVideoExtensions] = useState([]); - const [downloads, setDownloads] = useState>({}) + const [download, setDownload] = useState(undefined) + const downloading = useRef(false); useEffect(() => { if (serverConfig) { @@ -41,28 +40,6 @@ export default function PlaylistTree(props: PlaylistTreeProps) { return noop; }, [serverConfig]); - - const setDownloadsInfo = useCallback((info: FileDownloadInfo) => { - if (info.finished == undefined && info.filesize == undefined) { - setDownloads((downloads) => { - downloads[info.download_id] = {filename: info.filename, filesize: 0}; - return {...downloads}; - }); - } else { - if (info.filesize != undefined) { - setDownloads((downloads) => { - downloads[info.download_id].filesize = info.filesize; - return {...downloads}; - }); - } else { - setDownloads((downloads) => { - delete downloads[info.download_id]; - return {...downloads}; - }); - } - } - }, []); - const getPlaylistItemById = useCallback((itemId: string): PlaylistItem => { const id = parseInt(itemId); if (data && !isNaN(id)) { @@ -103,16 +80,28 @@ export default function PlaylistTree(props: PlaylistTreeProps) { } }, [enqueueSnackbar, getPlaylistItemById, onCopy]); - const startPollingDownload = useCallback((downloadId: string) => { - let subs: Subscription = services.file().getDownloadInfo(downloadId).subscribe({ - next: (info: FileDownloadInfo) => setDownloadsInfo(info), - error: (err) => enqueueSnackbar("Download file failed!", {variant: 'error'}), - complete: () => subs.unsubscribe() + const startPollingDownload = useCallback(() => { + let subs: Subscription = services.file().getDownloadInfo().subscribe({ + next: (info: FileDownloadInfo) => { + if (info.finished === true) { + setDownload(undefined); + } else { + setDownload(info); + } + }, + error: (err) => { + enqueueSnackbar("Download file failed!", {variant: 'error'}); + setDownload(undefined); + }, + complete: () => { + subs.unsubscribe(); + setDownload(undefined); + } }); - }, [setDownloadsInfo, enqueueSnackbar, services]); + }, [setDownload, enqueueSnackbar, services]); const handleDownloadUrl = useCallback((e: any) => { - if (! serverConfig.video.download?.directory) { + if (!serverConfig.video.download?.directory) { enqueueSnackbar("Please updated the server configuration and add video.download directory and headers!", {variant: 'error'}) } else { const item = getPlaylistItemById(e.target.dataset.item); @@ -125,11 +114,14 @@ export default function PlaylistTree(props: PlaylistTreeProps) { } if (VALID_VIDEO_FILES.includes(ext)) { - filename = filename + '.' + ext; - services.file().download({url: item.header.url, filename}).pipe(first()).subscribe({ + services.file().download({ + url: item.header.url, + filename: filename + '.' + ext + }).pipe(first()).subscribe({ next: (download: FileDownloadResponse) => { - setDownloadsInfo({download_id: download.download_id, filename: filename}); - startPollingDownload(download.download_id) + if (download.success) { + startPollingDownload(); + } }, error: err => enqueueSnackbar("Download failed!", {variant: 'error'}), complete: noop, @@ -139,7 +131,7 @@ export default function PlaylistTree(props: PlaylistTreeProps) { } } } - }, [serverConfig, enqueueSnackbar, getPlaylistItemById, services, startPollingDownload, setDownloadsInfo]); + }, [serverConfig, enqueueSnackbar, getPlaylistItemById, services, startPollingDownload]); const handlePlayUrl = useCallback((e: any) => { if (onPlay) { @@ -151,14 +143,14 @@ export default function PlaylistTree(props: PlaylistTreeProps) { }, [onPlay, getPlaylistItemById]); const isVideoFile = useCallback((entry: PlaylistItem): boolean => { - if (videoExtensions && entry.header.url) { - for (const ext of videoExtensions) { - if (entry.header.url.endsWith(ext)) { - return true; - } + if (videoExtensions && entry.header.url) { + for (const ext of videoExtensions) { + if (entry.header.url.endsWith(ext)) { + return true; } } - return false; + } + return false; }, [videoExtensions]); const renderEntry = useCallback((entry: PlaylistItem, index: number): React.ReactNode => { @@ -211,16 +203,16 @@ export default function PlaylistTree(props: PlaylistTreeProps) { }, [data, renderGroup]); const renderDownloads = useCallback((): React.ReactNode => { - const keys = Object.keys(downloads) - if (keys.length) { - let elements = keys.map(key => { - const info: DownloadInfo = downloads[key]; - return
  • {info.filename}: {info.filesize ? (info.filesize / 1_048_576).toFixed(2) : 0} MB
    • ; - }) - return
      {elements}
    ; + if (download) { + return
    +
      +
    • {download.filename}: {download.filesize ? (download.filesize / 1_048_576).toFixed(2) : 0} MB
      • + {download.errors?.length &&
    • {download.errors}
      • } +
    +
    ; } return <>; - }, [downloads]); + }, [download]); return
    {renderPlaylist()}{renderDownloads()}
    ; } \ No newline at end of file diff --git a/frontend/src/model/file-download.ts b/frontend/src/model/file-download.ts index 9fa50f82a..4616344bd 100644 --- a/frontend/src/model/file-download.ts +++ b/frontend/src/model/file-download.ts @@ -4,12 +4,18 @@ export interface FileDownloadRequest { } export interface FileDownloadResponse { - download_id: string; + success: boolean; } +export interface DownloadErrorInfo { + filename: string; + error: string; +} + + export interface FileDownloadInfo { - download_id: string; filename?: string; finished?: boolean; filesize?: number; + errors?: DownloadErrorInfo[]; } \ No newline at end of file diff --git a/frontend/src/service/file-service.ts b/frontend/src/service/file-service.ts index bee31fcc0..fd7703413 100755 --- a/frontend/src/service/file-service.ts +++ b/frontend/src/service/file-service.ts @@ -24,8 +24,8 @@ export default class FileService { return this.fileApiService.download(req); } - getDownloadInfo(download_id: string): Observable { - return this.fileApiService.getDownloadInfo(download_id); + getDownloadInfo(): Observable { + return this.fileApiService.getDownloadInfo(); } } diff --git a/src/api/api_model.rs b/src/api/api_model.rs index 5e5d7765d..e5f9a07d8 100644 --- a/src/api/api_model.rs +++ b/src/api/api_model.rs @@ -1,15 +1,37 @@ -use std::collections::HashMap; -use std::sync::{Arc, Mutex}; +use std::path::{PathBuf}; +use std::sync::{Arc, Mutex, RwLock}; use actix_web::web; use serde::{Deserialize, Serialize}; use crate::model::api_proxy::{ApiProxyConfig}; use crate::model::config::{Config, ConfigTargetOptions, ConfigRename, ConfigSort, InputType, ProcessTargets, TargetOutput, VideoConfig}; use crate::model::model_config::{default_as_empty_str, ProcessingOrder}; +#[derive(Serialize, Deserialize)] +pub(crate) struct DownloadErrorInfo { + pub filename: String, + pub error: String, +} + +#[derive(Clone)] +pub(crate) struct FileDownload { + pub file_dir: PathBuf, + pub file_path: PathBuf, + pub filename: String, + pub url: reqwest::Url, + pub size: u64, + pub error: Option, +} + +pub(crate) struct DownloadQueue { + pub queue: Arc>>, + pub active: Arc>>, + pub errors: Arc>>, +} + pub(crate) struct AppState { pub config: Arc, pub targets: Arc, - pub downloads: Arc>> + pub downloads: Arc } #[derive(Serialize)] diff --git a/src/api/main_api.rs b/src/api/main_api.rs index 8baaf05ee..c58ca5276 100644 --- a/src/api/main_api.rs +++ b/src/api/main_api.rs @@ -1,7 +1,6 @@ -use std::collections::HashMap; use std::io::ErrorKind; use std::path::PathBuf; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, Mutex, RwLock}; use actix_cors::Cors; use actix_files::NamedFile; @@ -9,7 +8,7 @@ use actix_web::{App, get, HttpRequest, HttpServer, web}; use actix_web::middleware::Logger; use crate::api::m3u_api::{m3u_api_register}; -use crate::api::api_model::{AppState}; +use crate::api::api_model::{AppState, DownloadQueue}; use crate::api::scheduler::start_scheduler; use crate::api::v1_api::{v1_api_register}; use crate::api::xmltv_api::{xmltv_api_register}; @@ -41,7 +40,11 @@ pub(crate) async fn start_server(cfg: Arc, targets: Arc) let shared_data = web::Data::new(AppState { config: cfg, targets, - downloads: Arc::new(Mutex::new(HashMap::new())) + downloads: Arc::from(DownloadQueue { + queue: Arc::from(Mutex::new(Vec::new())), + active: Arc::from(RwLock::new(None)), + errors: Arc::from(RwLock::new(Vec::new())), + }) }); // Scheduler diff --git a/src/api/v1_api.rs b/src/api/v1_api.rs index 11a1d7aa3..354cb47d8 100644 --- a/src/api/v1_api.rs +++ b/src/api/v1_api.rs @@ -1,21 +1,19 @@ -use std::collections::HashMap; use std::fs::File; use std::{fs, io}; use std::ffi::OsStr; use std::io::{ErrorKind, Write}; +use std::ops::Deref; use std::path::{Path, PathBuf}; -use std::sync::{Arc, Mutex}; +use std::sync::{Arc, RwLock}; use actix_web::{HttpResponse, Scope, web}; -use serde_json::json; -use uuid::Uuid; -use crate::api::api_model::{AppState, FileDownloadRequest, PlaylistRequest, ServerConfig, ServerInputConfig, ServerSourceConfig, ServerTargetConfig}; +use serde_json::{json}; +use crate::api::api_model::{AppState, DownloadErrorInfo, DownloadQueue, FileDownload, FileDownloadRequest, PlaylistRequest, ServerConfig, ServerInputConfig, ServerSourceConfig, ServerTargetConfig}; use crate::download::{get_m3u_playlist, get_xtream_playlist}; use crate::model::config::{ConfigInput, InputType, validate_targets, VideoDownloadConfig}; -use crate::utils::{bytes_to_megabytes}; +use crate::utils::{bytes_to_megabytes, get_request_headers}; use futures::stream::TryStreamExt; use log::{error, info}; -use reqwest::{header, Response}; -use reqwest::header::{HeaderName, HeaderValue}; +use reqwest::header::HeaderMap; use unidecode::unidecode; use crate::config_reader::save_api_proxy; use crate::m3u_filter_error::M3uFilterError; @@ -169,55 +167,106 @@ pub(crate) async fn config( HttpResponse::Ok().json(result) } -async fn async_download_file(download_id: &String, path: &Path, response: Response, downloads: Arc>>) -> Result { - match File::create(path) { - Ok(mut file) => { - info!("Downloading {}", path.to_str().unwrap_or("?")); - let mut stream = response.bytes_stream().map_err(|err| io::Error::new(ErrorKind::Other, err)); - let mut downloaded: u64 = 0; - downloads.lock().unwrap().insert(download_id.to_owned(), downloaded); - loop { - match stream.try_next().await { - Ok(item) => { - match item { - Some(chunk) => { - match file.write_all(&chunk) { - Ok(_) => { - let new = downloaded + (chunk.len() as u64); - downloaded = new; - downloads.lock().unwrap().insert(download_id.to_owned(), downloaded); +async fn download_file(active: Arc>>, headers: HeaderMap) -> Result<(), String> { + let client = reqwest::Client::new(); + let file_download = active.read().unwrap().as_ref().unwrap().clone(); + match client.get(file_download.url.clone()).headers(headers).send().await { + Ok(response) => { + match fs::create_dir_all(&file_download.file_dir) { + Ok(_) => { + let file_path_str = file_download.file_path.to_str().unwrap_or("?"); + match File::create(&file_download.file_path) { + Ok(mut file) => { + info!("Downloading {}", file_download.file_path.to_str().unwrap_or("?")); + let mut stream = response.bytes_stream().map_err(|err| io::Error::new(ErrorKind::Other, err)); + let mut downloaded: u64 = 0; + loop { + match stream.try_next().await { + Ok(item) => { + match item { + Some(chunk) => { + match file.write_all(&chunk) { + Ok(_) => { + downloaded += (chunk.len() as u64); + active.write().unwrap().as_mut().unwrap().size = downloaded; + } + Err(err) => return Err(format!("Error while writing to file: {} {}", file_path_str, err)) + } + } + None => { + let megabytes = bytes_to_megabytes(downloaded); + info!("Downloaded {}, filesize: {}MB", file_path_str, megabytes); + active.write().unwrap().as_mut().unwrap().size = downloaded; + return Ok(()); + } + } } - Err(err) => return Err(format!("Error while writing to file: {} {}", path.to_str().unwrap_or("?"), err)) + Err(err) => return Err(format!("Error while writing to file: {} {}", file_path_str, err)) } } - None => { - let megabytes = bytes_to_megabytes(downloaded); - info!("Downloaded {}, filesize: {}MB", path.to_str().unwrap_or("?"), megabytes); - return Ok(downloaded); - } } + Err(err) => Err(format!("Error while writing to file: {} {}", file_path_str, err)) } - Err(err) => return Err(format!("Error while writing to file: {} {}", path.to_str().unwrap_or("?"), err)) } + Err(err) => Err(format!("Error while creating directory to file: {} {}", &file_download.file_dir.to_str().unwrap_or("?"), err)) } } - Err(err) => Err(format!("Error while writing to file: {} {}", path.to_str().unwrap_or("?"), err)) + Err(err) => Err(format!("Error while opening url: {} {}", &file_download.url, err)) } } pub(crate) async fn download_file_info( - info: web::Path, _app_state: web::Data, ) -> HttpResponse { - let did: String = info.into_inner(); - match _app_state.downloads.lock().unwrap().get(&did) { - // @TODO it is only a success when the file remains. - None => HttpResponse::Ok().json(json!({"download_id": &did, "finished": true})), - Some(downloaded) => HttpResponse::Ok().json(json!({"download_id": &did, "filesize": downloaded})) + let error_list: &[DownloadErrorInfo] = &_app_state.downloads.errors.write().unwrap().drain(..) + .map(|e| DownloadErrorInfo { filename: e.filename, error: e.error.unwrap() }).collect::>(); + let errors = match serde_json::to_string(error_list) { + Ok(value) => value, + Err(_) => "[]".to_string() + }; + match &*_app_state.downloads.active.read().unwrap() { + None => HttpResponse::Ok().json(json!({"finished": true, "errors": errors})), + Some(file_download) => + HttpResponse::Ok().json(json!({"filename": file_download.filename, "filesize": file_download.size, "errors": errors})) } } -pub(crate) async fn download_file( +fn run_download_queue(download_cfg: &VideoDownloadConfig, download_queue: Arc) { + if let Some(file_download) = download_queue.as_ref().queue.lock().unwrap().pop() { + *download_queue.as_ref().active.write().unwrap() = Some(file_download); + let headers = get_request_headers(&download_cfg.headers); + let dq = Arc::clone(&download_queue); + actix_rt::spawn(async move { + loop { + let opt: Option = { + dq.active.read().unwrap().deref().clone() + }; + match opt { + Some(_) => { + match download_file(Arc::clone(&dq.active), headers.clone()).await { + Ok(_) => { + *dq.active.write().unwrap() = dq.queue.lock().unwrap().pop(); + } + Err(err) => { + if let Some(fd) = &mut *dq.active.write().unwrap() { + fd.error = Some(err); + dq.errors.write().unwrap().push(fd.clone()); + } + *dq.active.write().unwrap() = dq.queue.lock().unwrap().pop(); + } + } + } + None => { + return; + } + } + } + }); + } +} + + +pub(crate) async fn queue_download_file( req: web::Json, _app_state: web::Data, ) -> HttpResponse { @@ -228,49 +277,25 @@ pub(crate) async fn download_file( match reqwest::Url::parse(&req.url) { Ok(_url) => { - let client = reqwest::Client::new(); - let mut headers = header::HeaderMap::new(); - for (key, value) in &download.headers { - headers.insert( - HeaderName::from_bytes(key.as_bytes()).unwrap(), - HeaderValue::from_bytes(value.as_bytes()).unwrap(), - ); + let filename_re = download._re_filename.as_ref().unwrap(); + let filename = filename_re.replace_all(&unidecode(&req.filename).replace(' ', "_"), "").to_string(); + let file_name = filename.clone(); + let file_dir = get_download_directory(download, &filename); + let mut file_path: PathBuf = file_dir.clone(); + file_path.push(&filename); + let file_download = FileDownload { + file_dir, + file_path, + filename, + url: _url, + size: 0, + error: None, + }; + _app_state.downloads.queue.lock().unwrap().push(file_download); + if _app_state.downloads.active.read().unwrap().is_none() { + run_download_queue(download, Arc::clone(&_app_state.downloads)); } - match client.get(_url).headers(headers).send().await { - Ok(response) => { - let filename_re = download._re_filename.as_ref().unwrap(); - let filename = filename_re.replace_all(&unidecode(&req.filename).replace(' ', "_"), "").to_string(); - let file_dir = get_download_directory(download, &filename); - match fs::create_dir_all(&file_dir) { - Ok(_) => { - let path = file_dir.join(filename.as_str()); - let download_id = Uuid::new_v4().to_string(); - let response_download_id = download_id.clone(); - actix_rt::spawn(async move { - let downloads = _app_state.downloads.clone(); - match async_download_file(&download_id, &path, response, downloads.clone()).await { - Ok(_) => { - downloads.lock().unwrap().remove(&download_id); - } - Err(err) => { - downloads.lock().unwrap().remove(&download_id); - let _ = fs::remove_file(&path); - error!("{}", err); - } - } - }); - HttpResponse::Ok().json(json!({"download_id": response_download_id})) - } - Err(err) => HttpResponse::InternalServerError().json(json!({"error": format!("{}", err)})) - } - } - Err(err) => HttpResponse::InternalServerError().json(json!({"error": format!("{}", err)})), - } - - // - // use rocket::futures::TryStreamExt; // for map_err() call below: - // let reader = StreamReader::new(response.bytes_stream().map_err(|e| std::io::Error::new(std::io::ErrorKind::Other, e))); - // rocket::response::Stream::chunked(reader, 4096) + HttpResponse::Ok().json(json!({"success": file_name})) } Err(_) => HttpResponse::BadRequest().json(json!({"error": "Invalid Arguments"})), } @@ -307,6 +332,6 @@ pub(crate) fn v1_api_register() -> Scope { .route("/config/serverinfo", web::post().to(config_api_proxy_server_info)) .route("/playlist", web::post().to(playlist)) .route("/playlist/update", web::post().to(playlist_update)) - .route("/file/download", web::post().to(download_file)) - .route("/file/download/{did}", web::get().to(download_file_info)) + .route("/file/download", web::post().to(queue_download_file)) + .route("/file/download/info", web::get().to(download_file_info)) } diff --git a/src/utils.rs b/src/utils.rs index 22992939b..9d7101678 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,10 +1,11 @@ +use std::collections::HashMap; use std::fs; use std::io::{Read, Write}; use std::path::{Path, PathBuf}; use log::{debug, error}; use path_absolutize::*; use reqwest::header; -use reqwest::header::{HeaderName, HeaderValue}; +use reqwest::header::{HeaderMap, HeaderName, HeaderValue}; use crate::create_m3u_filter_error_result; use crate::m3u_filter_error::{M3uFilterError, M3uFilterErrorKind}; use crate::model::config::{ConfigInput}; @@ -198,19 +199,24 @@ pub(crate) fn get_file_path(wd: &String, path: Option) -> Option reqwest::RequestBuilder { let mut request = reqwest::Client::new().get(url); if input.headers.is_empty() { - let mut headers = header::HeaderMap::new(); - for (key, value) in &input.headers { - headers.insert( - HeaderName::from_bytes(key.as_bytes()).unwrap(), - HeaderValue::from_bytes(value.as_bytes()).unwrap(), - ); - } - debug!("Request with headers{:?}", &headers); + let headers = get_request_headers(&input.headers); request = request.headers(headers); } request } +pub fn get_request_headers(defined_headers: &HashMap) -> HeaderMap { + let mut headers = header::HeaderMap::new(); + for (key, value) in defined_headers { + headers.insert( + HeaderName::from_bytes(key.as_bytes()).unwrap(), + HeaderValue::from_bytes(value.as_bytes()).unwrap(), + ); + } + debug!("Request with headers{:?}", &headers); + headers +} + async fn download_json_content(input: &ConfigInput, url: url::Url, persist_filepath: Option) -> Result { let request = get_client_request(input, url); match request.send().await { @@ -284,4 +290,4 @@ pub(crate) fn path_exists(file_path: &Path) -> bool { return metadata.is_file() } false -} \ No newline at end of file +}