diff --git a/setup.py b/setup.py
index 761d552c..c4a55092 100755
--- a/setup.py
+++ b/setup.py
@@ -435,6 +435,11 @@ class pil_build_ext(build_ext):
                 _add_directory(include_dirs,
                                os.path.join(best_path, 'include'))
 
+        # The logic above fails in many cases (such as pypy builds) and we fix
+        # it here by making sure we look in standard places first.
+        library_dirs = ['/usr/lib64', '/usr/lib'] + library_dirs
+        include_dirs = ['/usr/include'] + include_dirs
+
         #
         # insert new dirs *before* default libs, to avoid conflicts
         # between Python PYD stub libs and real libraries