diff --git a/README.md b/README.md
index 7608dac..17be96f 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ Install with PIP
 pip3 install torch torchvision --extra-index-url https://download.pytorch.org/whl/cu117
 
 # install clip-interrogator
-pip install clip-interrogator==0.5.2
+pip install clip-interrogator==0.5.3
 ```
 
 You can then use it in your script
diff --git a/clip_interrogator/clip_interrogator.py b/clip_interrogator/clip_interrogator.py
index f0fca31..cb7b0a0 100644
--- a/clip_interrogator/clip_interrogator.py
+++ b/clip_interrogator/clip_interrogator.py
@@ -300,7 +300,7 @@ class Interrogator():
             self.blip_offloaded = False
 
     def _prepare_clip(self):
-        if self.config.blip_offload:
+        if self.config.blip_offload and not self.blip_offloaded:
             self.blip_model = self.blip_model.to('cpu')
             self.blip_offloaded = True
         if self.clip_offloaded:
diff --git a/setup.py b/setup.py
index d6ce654..b3297fc 100644
--- a/setup.py
+++ b/setup.py
@@ -5,7 +5,7 @@ from setuptools import setup, find_packages
 
 setup(
     name="clip-interrogator",
-    version="0.5.2",
+    version="0.5.3",
     license='MIT',
     author='pharmapsychotic',
     author_email='me@pharmapsychotic.com',