Multiprocessing 8

file_list.txt

@@ -48,9 +48,6 @@ demo_data/demo/2/43789_right.jpeg
 demo_data/demo/2/7906_right.jpeg
 demo_data/demo/2/31160_left.jpeg
 demo_data/demo/2/7805_right.jpeg
-release_ckpts/2023-12-20_18-39-06_30301601_V100_x4_resnet50_896_setseeds/inference/preprocessor_config.json
-release_ckpts/2023-12-20_18-39-06_30301601_V100_x4_resnet50_896_setseeds/inference/config.json
-release_ckpts/2023-12-20_18-39-06_30301601_V100_x4_resnet50_896_setseeds/inference/model.safetensors
 release_ckpts/2023-12-24_20-02-18_30345221_V100_x4_resnet34/inference/preprocessor_config.json
 release_ckpts/2023-12-24_20-02-18_30345221_V100_x4_resnet34/inference/model.safetensors
 release_ckpts/2023-12-24_20-02-18_30345221_V100_x4_resnet34/inference/config.json

src/fetch_files.py

@@ -1,33 +1,43 @@
 import os
 import requests
+import multiprocessing
 
 REPO_ROOT = os.path.realpath(os.path.join(os.path.split(__file__)[0], ".."))
 
 
-def fetch_files():
+def fetch_one(subpath: str) -> None:
     s3_url = "https://sdaia-kaust-public.s3.us-east-2.amazonaws.com/diabetic-retinopathy-detection/"
+
+    url = f"{s3_url}{subpath}"
+    target_path = os.path.join(REPO_ROOT, subpath)
+    if os.path.exists(target_path):
+        print(f"File already in place: {target_path}")
+        return
+    print(f"Downloading {url}...")
+    req = requests.get(url)
+    if not req.ok:
+        print(f"Failed to download {url}")
+        return
+    print(f"Downloaded {url}")
+    target_dir = os.path.split(target_path)[0]
+    os.makedirs(target_dir, exist_ok=True)
+    with open(target_path, "wb") as fdst:
+        fdst.write(req.content)
+    print(f"Saved to {target_path}")
+
+
+def fetch_files():
     flist_path = os.path.join(REPO_ROOT, "file_list.txt")
+    subpaths = []
     with open(flist_path, "r") as file:
         for line in file:
             subpath = line.strip()
-            url = f"{s3_url}{subpath}"
-            target_path = os.path.join(REPO_ROOT, subpath)
-            if os.path.exists(target_path):
-                print(f"File already in place: {target_path}")
-                continue
-            print(f"Downloading {url}...")
-            req = requests.get(url)
-            if not req.ok:
-                print(f"Failed to download {url}")
-                continue
-            print(f"Downloaded {url}")
-            target_dir = os.path.split(target_path)[0]
-            os.makedirs(target_dir, exist_ok=True)
-            with open(target_path, "wb") as fdst:
-                fdst.write(req.content)
-            print(f"Saved to {target_path}")
+            subpaths.append(subpath)
+
+    with multiprocessing.Pool(8) as pool:
+        pool.map(fetch_one, subpaths)
+
     print("File fetching done")
-    return
 
 
 if __name__ == "__main__":