From c83e47d3b13353817a146970b83deac3080452b6 Mon Sep 17 00:00:00 2001 From: Samer Afach Date: Sun, 13 Nov 2016 12:36:39 +0100 Subject: [PATCH] Added exception error message to the output message on failure of multiprocessing.cpu_count(). --- listdownloader/downloader.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/listdownloader/downloader.py b/listdownloader/downloader.py index a87f2b6..9d104ba 100644 --- a/listdownloader/downloader.py +++ b/listdownloader/downloader.py @@ -171,17 +171,23 @@ def download_files(list_of_urls, to_dir, processes=0): list_of_urls = [line.replace(' ', '').replace('\n', '').replace('\t', '') for line in list_of_urls] if not os.path.isdir(to_dir): mkdir_p(to_dir) + + # try to detect the number of CPUs automatically if processes <= 0: try: processes = mp.cpu_count() except NotImplementedError as e: sys.stderr.write("Unable to determine the number of CPUs for parallelization. Proceeding sequentially. " - "Consider inputting the number of CPUs manually.\n") + "Consider inputting the number of CPUs manually. The error says: " + str(e) + "\n") _download_files(list_of_urls, to_dir) return + + # if there's only 1 process or 1 URL, there's no need to use multiprocessing elif processes == 1 or len(list_of_urls) == 1: _download_files(list_of_urls, to_dir) return + + # if number of processes is larger than the number of URLs, reduce the number of processes to save resources elif processes > len(list_of_urls): processes = len(list_of_urls) @@ -189,4 +195,4 @@ def download_files(list_of_urls, to_dir, processes=0): pool = mp.Pool(processes) pool.starmap(download_file, params) pool.close() - pool.join() \ No newline at end of file + pool.join()