CMake: use pkg-config and Homebrew when looking for libcrypto.

Do the same sort of stuff that configure does, but CMake-style.

(cherry picked from commit fc8465a)

CMakeLists.txt

@@ -130,6 +130,80 @@ project(tcpdump C)
 #
 set(SIZEOF_VOID_P "${CMAKE_SIZEOF_VOID_P}")
 
+#
+# Show the bit width for which we're compiling.
+# This can help debug problems if you're dealing with a compiler that
+# defaults to generating 32-bit code even when running on a 64-bit
+# platform, and where that platform may provide only 64-bit versions of
+# libraries that we might use (looking at *you*, Oracle Studio!).
+#
+if(CMAKE_SIZEOF_VOID_P EQUAL 4)
+  message(STATUS "Building 32-bit")
+elseif(CMAKE_SIZEOF_VOID_P EQUAL 8)
+  message(STATUS "Building 64-bit")
+endif()
+
+#
+# Solaris pkg-config is annoying.  For at least one package (D-Bus, I'm
+# looking at *you*!), there are separate include files for 32-bit and
+# 64-bit builds (I guess using "unsigned long long" as a 64-bit integer
+# type on a 64-bit build is like crossing the beams or something), and
+# there are two separate .pc files, so if we're doing a 32-bit build we
+# should make sure we look in /usr/lib/pkgconfig for .pc files and if
+# we're doing a 64-bit build we should make sure we look in
+# /usr/lib/amd64/pkgconfig for .pc files.
+#
+if(CMAKE_SYSTEM_NAME STREQUAL "SunOS" AND CMAKE_SYSTEM_VERSION MATCHES "5[.][0-9.]*")
+    #
+    # Note: string(REPLACE) does not appear to support using ENV{...}
+    # as an argument, so we set a variable and then use set() to set
+    # the environment variable.
+    #
+    if(CMAKE_SIZEOF_VOID_P EQUAL 8)
+        #
+        # 64-bit build.  If /usr/lib/pkgconfig appears in the path,
+        # prepend /usr/lib/amd64/pkgconfig to it; otherwise,
+        # put /usr/lib/amd64 at the end.
+        #
+        if((NOT DEFINED ENV{PKG_CONFIG_PATH}) OR "$ENV{PKG_CONFIG_PATH}" EQUAL "")
+            #
+            # Not set, or empty.  Set it to /usr/lib/amd64/pkgconfig.
+            #
+            set(fixed_path "/usr/lib/amd64/pkgconfig")
+        elseif("$ENV{PKG_CONFIG_PATH}" MATCHES "/usr/lib/pkgconfig")
+            #
+            # It contains /usr/lib/pkgconfig.  Prepend
+            # /usr/lib/amd64/pkgconfig to /usr/lib/pkgconfig.
+            #
+            string(REPLACE "/usr/lib/pkgconfig"
+                "/usr/lib/amd64/pkgconfig:/usr/lib/pkgconfig"
+                fixed_path "$ENV{PKG_CONFIG_PATH}")
+        else()
+            #
+            # Not empty, but doesn't contain /usr/lib/pkgconfig.
+            # Append /usr/lib/amd64/pkgconfig to it.
+            #
+            set(fixed_path "$ENV{PKG_CONFIG_PATH}:/usr/lib/amd64/pkgconfig")
+        endif()
+        set(ENV{PKG_CONFIG_PATH} "${fixed_path}")
+    elseif(CMAKE_SIZEOF_VOID_P EQUAL 4)
+        #
+        # 32-bit build.  If /usr/amd64/lib/pkgconfig appears in the path,
+        # prepend /usr/lib/pkgconfig to it.
+        #
+        if("$ENV{PKG_CONFIG_PATH}" MATCHES "/usr/lib/amd64/pkgconfig")
+            #
+            # It contains /usr/lib/amd64/pkgconfig.  Prepend
+            # /usr/lib/pkgconfig to /usr/lib/amd64/pkgconfig.
+            #
+            string(REPLACE "/usr/lib/amd64/pkgconfig"
+                "/usr/lib/pkgconfig:/usr/lib/amd64/pkgconfig"
+                fixed_path "$ENV{PKG_CONFIG_PATH}")
+            set(ENV{PKG_CONFIG_PATH} "${fixed_path}")
+        endif()
+    endif()
+endif()
+
 #
 # For checking if a compiler flag works and adding it if it does.
 #
@@ -902,11 +976,6 @@ endif(WITH_SMI)
 if(WITH_CRYPTO)
     find_package(CRYPTO)
     if(CRYPTO_FOUND)
-        #
-        # Check for some headers and functions.
-        #
-        check_include_file(openssl/evp.h HAVE_OPENSSL_EVP_H)
-
         #
         # 1) do we have EVP_CIPHER_CTX_new?
         # If so, we use it to allocate an EVP_CIPHER_CTX, as

cmake/Modules/FindCRYPTO.cmake

@@ -2,23 +2,186 @@
 # Try to find libcrypto.
 #
 
-# Try to find the header
-find_path(CRYPTO_INCLUDE_DIR openssl/crypto.h)
+#
+# Were we told where to look for libcrypto?
+#
+if(NOT CRYPTO_ROOT)
+  #
+  # No.
+  #
+  # First, try looking for it with pkg-config, if we have it.
+  #
+  find_package(PkgConfig)
+
+  #
+  # Homebrew's pkg-config does not, by default, look for
+  # pkg-config files for packages it has installed.
+  # Furthermore, at least for OpenSSL, they appear to be
+  # dumped in package-specific directories whose paths are
+  # not only package-specific but package-version-specific.
+  #
+  # So the only way to find openssl is to get the value of
+  # PKG_CONFIG_PATH from "brew --env openssl" and add that
+  # to PKG_CONFIG_PATH.  (No, we can't just assume it's under
+  # /usr/local; Homebrew have conveniently chosen to put it
+  # under /opt/homebrew on ARM.)
+  #
+  # That's the nice thing about Homebrew - it makes things easier!
+  # Thanks!
+  #
+  find_program(BREW brew)
+  if(BREW)
+    #
+    # We have Homebrew.
+    # Get the pkg-config directory for openssl.
+    #
+    execute_process(COMMAND "${BREW}" "--env" "--plain" "openssl"
+      RESULT_VARIABLE BREW_RESULT
+      OUTPUT_VARIABLE BREW_OUTPUT
+      OUTPUT_STRIP_TRAILING_WHITESPACE
+    )
+    if(BREW_RESULT EQUAL 0)
+      #
+      # brew --env --plain openssl succeeded.
+      # Split its output into a list, one entry per line.
+      #
+      string(REGEX MATCHALL "[^\n\r]+" BREW_OUTPUT_LINES "${BREW_OUTPUT}")
+
+      #
+      # Find the line that begins with "PKG_CONFIG_PATH: ", and extract
+      # the path following that.
+      #
+      foreach(LINE IN LISTS BREW_OUTPUT_LINES)
+        if(LINE MATCHES "PKG_CONFIG_PATH: \(.*\)")
+          string(REGEX REPLACE "PKG_CONFIG_PATH: \(.*\)"
+              "\\1" OPENSSL_PKGCONFIG_DIR
+              ${LINE})
+        endif()
+      endforeach()
+    endif()
+  endif()
+
+  #
+  # Save the current value of the PKG_CONFIG_PATH environment
+  # variable.
+  #
+  set(SAVE_PKG_CONFIG_PATH $ENV{PKG_CONFIG_PATH})
+
+  #
+  # If we got an additional pkg-config directory from Homebrew, add
+  # it to the PKG_CONFIG_PATH environment variable.
+  #
+  if(OPENSSL_PKGCONFIG_DIR)
+    set(ENV{PKG_CONFIG_PATH} "${OPENSSL_PKGCONFIG_DIR}:$ENV{PKG_CONFIG_PATH}")
+  endif()
+
+  #
+  # Use pkg-config to find libcrypto.
+  #
+  pkg_check_modules(CRYPTO libcrypto)
 
-# Try to find the library
-find_library(CRYPTO_LIBRARY crypto)
+  #
+  # Revert the change to PKG_CONFIG_PATH.
+  #
+  set(ENV{PKG_CONFIG_PATH} "${SAVE_PKG_CONFIG_PATH}")
+
+  #
+  # Did pkg-config find it?
+  #
+  if(CRYPTO_FOUND)
+    #
+    # This "helpfully" supplies CRYPTO_LIBRARIES as a bunch of
+    # library names - not paths - and CRYPTO_LIBRARY_DIRS as
+    # a bunch of directories.
+    #
+    # CMake *really* doesn't like the notion of specifying "here are
+    # the directories in which to look for libraries" except in
+    # find_library() calls; it *really* prefers using full paths to
+    # library files, rather than library names.
+    #
+    # Find the libraries and add their full paths.
+    #
+    set(CRYPTO_LIBRARY_FULLPATHS)
+    foreach(_lib IN LISTS CRYPTO_LIBRARIES)
+      #
+      # Try to find this library, so we get its full path.
+      #
+      find_library(_libfullpath ${_lib} HINTS ${CRYPTO_LIBRARY_DIRS})
+      list(APPEND CRYPTO_LIBRARY_FULLPATHS ${_libfullpath})
+    endforeach()
+    set(CRYPTO_LIBRARIES "${CRYPTO_LIBRARY_FULLPATHS}")
+  else()
+    #
+    # No.  If we have Homebrew installed, see if it's in Homebrew.
+    #
+    if(BREW)
+      #
+      # The brew man page lies when it speaks of
+      # $BREW --prefix --installed <formula>
+      # outputting nothing.  In Homebrew 3.3.16,
+      # it produces output regardless of whether
+      # the formula is installed or not, so we
+      # send the standard output and error to
+      # the bit bucket.
+      #
+      # libcrypto isn't a formula, openssl is a formula.
+      #
+      execute_process(COMMAND "${BREW}" "--prefix" "--installed" "openssl"
+        RESULT_VARIABLE BREW_RESULT
+        OUTPUT_QUIET
+      )
+      if(BREW_RESULT EQUAL 0)
+        #
+        # Yes.  Get the include directory and library
+        # directory.  (No, we can't just assume it's
+        # under /usr/local; Homebrew have conveniently
+        # chosen to put it under /opt/homebrew on ARM.)
+        #
+        execute_process(COMMAND "${BREW}" "--prefix" "openssl"
+          RESULT_VARIABLE BREW_RESULT
+          OUTPUT_VARIABLE OPENSSL_PATH
+          OUTPUT_STRIP_TRAILING_WHITESPACE
+        )
+        set(CRYPTO_INCLUDE_DIRS "${OPENSSL_PATH}/include")
+
+        #
+        # Search for the libcrypto library under lib.
+        #
+        find_library(CRYPTO_LIBRARIES crypto
+            PATHS "${OPENSSL_PATH}/lib"
+            NO_DEFAULT_PATH)
+      endif()
+    endif()
+  endif()
+endif()
+
+#
+# Have we found it with pkg-config or Homebrew?
+#
+if(NOT CRYPTO_INCLUDE_DIRS)
+  #
+  # No.
+  # Try to find the openss/evp.h header.
+  # We search for that header to make sure that it's installed (if
+  # it's just a shared library for the benefit of existing
+  # programs, that's not useful).
+  #
+  find_path(CRYPTO_INCLUDE_DIRS openssl/evp.h)
+
+  #
+  # Try to find the library.
+  #
+  find_library(CRYPTO_LIBRARIES crypto)
+endif()
 
 include(FindPackageHandleStandardArgs)
 find_package_handle_standard_args(CRYPTO
   DEFAULT_MSG
-  CRYPTO_INCLUDE_DIR
-  CRYPTO_LIBRARY
+  CRYPTO_INCLUDE_DIRS
+  CRYPTO_LIBRARIES
 )
 
 mark_as_advanced(
-  CRYPTO_INCLUDE_DIR
-  CRYPTO_LIBRARY
+  CRYPTO_INCLUDE_DIRS
+  CRYPTO_LIBRARIES
 )
-
-set(CRYPTO_INCLUDE_DIRS ${CRYPTO_INCLUDE_DIR})
-set(CRYPTO_LIBRARIES ${CRYPTO_LIBRARY})