diff --git a/ax_check_zlib.m4 b/ax_check_zlib.m4
index d308183..07f4c25 100644
--- a/ax_check_zlib.m4
+++ b/ax_check_zlib.m4
@@ -65,12 +65,12 @@ AC_DEFUN([AX_CHECK_ZLIB],
 # Handle user hints
 #
 [AC_MSG_CHECKING(if zlib is wanted)
+zlib_places="/usr/local /usr /opt/local /sw"
 AC_ARG_WITH(zlib,
 [  --with-zlib=DIR root directory path of zlib installation [defaults to
                     /usr/local or /usr if not found in /usr/local]
   --without-zlib to disable zlib usage completely],
 [if test "$withval" != no ; then
-  zlib_places="/usr/local /usr /opt/local /sw"
   AC_MSG_RESULT(yes)
   if test -d "$withval"
   then
@@ -80,6 +80,7 @@ AC_ARG_WITH(zlib,
   fi
 else
   AC_MSG_RESULT(no)
+  zlib_places=
 fi],
 [AC_MSG_RESULT(yes)])
 
