Fetch checkpoints and images from AWS S3

.gitignore

@@ -4,6 +4,8 @@
 /slurm_logs
 
 /lightning_logs/
+/demo_data/
+/release_ckpts/
 
 __pycache__
 

app.py

@@ -10,6 +10,7 @@ from transformers import (AutoImageProcessor,
                           ResNetForImageClassification)
 
 from src.labelmap import DR_LABELMAP
+from src.fetch_files import fetch_files
 
 
 class App:
@@ -22,6 +23,9 @@ class App:
 
         path = f"release_ckpts/{ckpt_name}/inference/"
 
+        if not os.path.exists(path):
+            raise Exception(f"Checkpoint not found at {path}")
+
         self.image_processor = AutoImageProcessor.from_pretrained(path)
 
         self.model = ResNetForImageClassification.from_pretrained(path)
@@ -147,6 +151,7 @@ class App:
 
 def main():
     """ App entry point. """
+    fetch_files()
     app = App()
     app.launch()
 

file_list.txt

@@ -0,0 +1,56 @@
+demo_data/demo/0/11573_left.jpeg
+demo_data/demo/0/24879_left.jpeg
+demo_data/demo/0/30682_right.jpeg
+demo_data/demo/0/23654_left.jpeg
+demo_data/demo/0/25149_right.jpeg
+demo_data/demo/0/28148_left.jpeg
+demo_data/demo/0/24009_left.jpeg
+demo_data/demo/0/8594_right.jpeg
+demo_data/demo/0/2961_right.jpeg
+demo_data/demo/0/6420_left.jpeg
+demo_data/demo/3/30226_left.jpeg
+demo_data/demo/3/26254_right.jpeg
+demo_data/demo/3/34918_right.jpeg
+demo_data/demo/3/15930_right.jpeg
+demo_data/demo/3/12881_left.jpeg
+demo_data/demo/3/2577_left.jpeg
+demo_data/demo/3/14481_right.jpeg
+demo_data/demo/3/11219_left.jpeg
+demo_data/demo/3/11031_right.jpeg
+demo_data/demo/3/6433_left.jpeg
+demo_data/demo/4/38186_left.jpeg
+demo_data/demo/4/39505_left.jpeg
+demo_data/demo/4/19471_left.jpeg
+demo_data/demo/4/33906_right.jpeg
+demo_data/demo/4/13669_left.jpeg
+demo_data/demo/4/14436_right.jpeg
+demo_data/demo/4/37958_left.jpeg
+demo_data/demo/4/4130_left.jpeg
+demo_data/demo/4/4086_left.jpeg
+demo_data/demo/4/43839_right.jpeg
+demo_data/demo/1/42690_left.jpeg
+demo_data/demo/1/38114_right.jpeg
+demo_data/demo/1/41813_right.jpeg
+demo_data/demo/1/31010_right.jpeg
+demo_data/demo/1/25680_right.jpeg
+demo_data/demo/1/17629_left.jpeg
+demo_data/demo/1/40316_right.jpeg
+demo_data/demo/1/25154_right.jpeg
+demo_data/demo/1/30124_right.jpeg
+demo_data/demo/1/3782_left.jpeg
+demo_data/demo/2/25945_left.jpeg
+demo_data/demo/2/1407_right.jpeg
+demo_data/demo/2/5728_left.jpeg
+demo_data/demo/2/27297_left.jpeg
+demo_data/demo/2/12003_right.jpeg
+demo_data/demo/2/21188_right.jpeg
+demo_data/demo/2/43789_right.jpeg
+demo_data/demo/2/7906_right.jpeg
+demo_data/demo/2/31160_left.jpeg
+demo_data/demo/2/7805_right.jpeg
+release_ckpts/2023-12-20_18-39-06_30301601_V100_x4_resnet50_896_setseeds/inference/preprocessor_config.json
+release_ckpts/2023-12-20_18-39-06_30301601_V100_x4_resnet50_896_setseeds/inference/config.json
+release_ckpts/2023-12-20_18-39-06_30301601_V100_x4_resnet50_896_setseeds/inference/model.safetensors
+release_ckpts/2023-12-24_20-02-18_30345221_V100_x4_resnet34/inference/preprocessor_config.json
+release_ckpts/2023-12-24_20-02-18_30345221_V100_x4_resnet34/inference/model.safetensors
+release_ckpts/2023-12-24_20-02-18_30345221_V100_x4_resnet34/inference/config.json

src/fetch_files.py

@@ -0,0 +1,34 @@
+import os
+import requests
+
+REPO_ROOT = os.path.realpath(os.path.join(os.path.split(__file__)[0], ".."))
+
+
+def fetch_files():
+    s3_url = "https://sdaia-kaust-public.s3.us-east-2.amazonaws.com/diabetic-retinopathy-detection/"
+    flist_path = os.path.join(REPO_ROOT, "file_list.txt")
+    with open(flist_path, "r") as file:
+        for line in file:
+            subpath = line.strip()
+            url = f"{s3_url}{subpath}"
+            target_path = os.path.join(REPO_ROOT, subpath)
+            if os.path.exists(target_path):
+                print(f"File already in place: {target_path}")
+                continue
+            print(f"Downloading {url}...")
+            req = requests.get(url)
+            if not req.ok:
+                print(f"Failed to download {url}")
+                continue
+            print(f"Downloaded {url}")
+            target_dir = os.path.split(target_path)[0]
+            os.makedirs(target_dir, exist_ok=True)
+            with open(target_path, "wb") as fdst:
+                fdst.write(req.content)
+            print(f"Saved to {target_path}")
+    print("File fetching done")
+    return
+
+
+if __name__ == "__main__":
+    fetch_files()