Merge pull request #5414 from hppritcha/topic/iwarp_only_by_default

btl/openib: only look for iwarp/roce by default

Author: hppritcha

opal/mca/btl/openib/btl_openib.h

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2011 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
- * Copyright (c) 2006-2016 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2006-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
  * Copyright (c) 2009-2010 Oracle and/or its affiliates.  All rights reserved.
@@ -245,6 +245,7 @@ struct mca_btl_openib_component_t {
        guarantee about the size of an enum. this value will be registered as an
        integer with the MCA variable system */
     int device_type;
+    bool allow_ib;
     char *if_include;
     char **if_include_list;
     char *if_exclude;

opal/mca/btl/openib/btl_openib_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2018 Cisco Systems, Inc.  All rights reserved
  * Copyright (c) 2006-2015 Mellanox Technologies. All rights reserved.
- * Copyright (c) 2006-2015 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2006-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
  * Copyright (c) 2009-2012 Oracle and/or its affiliates.  All rights reserved.
@@ -622,6 +622,30 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_device_t *device,
     union ibv_gid gid;
     uint64_t subnet_id;
 
+/*
+ * Starting with Open MPI 4.0 we don't support infiniband
+ * unless the user specifically requested to override this
+ * policy.  For ancient OFED, only allow if user has set
+ * the MCA parameter.
+ */
+#if HAVE_DECL_IBV_LINK_LAYER_ETHERNET
+    if ((IBV_LINK_LAYER_INFINIBAND == ib_port_attr->link_layer) &&
+        (false == mca_btl_openib_component.allow_ib)) {
+        opal_show_help("help-mpi-btl-openib.txt", "ib port not selected",
+                       true, opal_process_info.nodename,
+                       ibv_get_device_name(device->ib_dev), port_num);
+        return OPAL_ERR_NOT_FOUND;
+    }
+#else
+    if (false == mca_btl_openib_component.allow_ib) {
+        opal_show_help("help-mpi-btl-openib.txt", "ib port not selected",
+                       true, opal_process_info.nodename,
+                       ibv_get_device_name(device->ib_dev), port_num);
+        return OPAL_ERR_NOT_FOUND;
+    }
+#endif
+
+
     /* Ensure that the requested GID index (via the
        btl_openib_gid_index MCA param) is within the GID table
        size. */

opal/mca/btl/openib/btl_openib_mca.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2006-2009 Mellanox Technologies. All rights reserved.
- * Copyright (c) 2006-2015 Los Alamos National Security, LLC.  All rights
+ * Copyright (c) 2006-2018 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2006-2007 Voltaire All rights reserved.
  * Copyright (c) 2009-2010 Oracle and/or its affiliates.  All rights reserved.
@@ -291,6 +291,19 @@ int btl_openib_register_mca_params(void)
     if (0 > tmp) ret = tmp;
     OBJ_RELEASE(new_enum);
 
+    /*
+     * Provide way for using to override policy of ignoring IB HCAs
+     */
+
+    mca_btl_openib_component.allow_ib = false;
+    tmp = mca_base_component_var_register(&mca_btl_openib_component.super.btl_version,
+                                          "allow_ib",
+                                          "Override policy since Open MPI 4.0 of ignoring IB HCAs for openib BTL",
+                                          MCA_BASE_VAR_TYPE_BOOL, NULL,
+                                          0, 0, OPAL_INFO_LVL_5,
+                                          MCA_BASE_VAR_SCOPE_READONLY,
+                                          &mca_btl_openib_component.allow_ib);
+
     CHECK(reg_int("max_btls", NULL,
                   "Maximum number of device ports to use "
                   "(-1 = use all available, otherwise must be >= 1)",

opal/mca/btl/openib/help-mpi-btl-openib.txt

@@ -14,6 +14,8 @@
 # Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved.
 # Copyright (c) 2009      Sun Microsystems, Inc.  All rights reserved.
 # Copyright (c) 2013-2014 NVIDIA Corporation.  All rights reserved.
+# Copyright (c) 2018      Los Alamos National Security, LLC.  All rights
+#                         reserved.
 # $COPYRIGHT$
 #
 # Additional copyrights may follow
@@ -250,6 +252,10 @@ this run.
 
 If the "requested type" is "<any>", this usually means that *no*
 OpenFabrics devices were found within the requested bus distance.
+
+Note starting with Open MPI 4.0, only iWarp and RoCE devices are considered
+for selection by default.  Set the btl_openib_allow_ib MCA
+parameter to "true" to allow use of Infiniband devices.
 #
 [default subnet prefix]
 WARNING: There are more than one active ports on host '%s', but the
@@ -706,3 +712,14 @@ Open MPI has detected that you have attempted to set the btl_openib_cuda_max_sen
 value. This is not supported. Setting back to default value of 0.
 
   Local host:              %s
+[ib port not selected]
+By default, for Open MPI 4.0 and later, infiniband ports on a device
+are not used by default.  The intent is to use UCX for these devices.
+You can override this policy by setting the btl_openib_allow_ib MCA parameter
+to true.
+
+  Local host:              %s
+  Local adapter:           %s
+  Local port:              %d
+#
+