created weight norm option (#809)

Summary:
Pull Request resolved: #809

Weight norm decouples a weight matrix into its magnitude and direction and optimizes them separately -- this is very helpful in speeding up convergence and getting better performance overall. This diff adds an option to use weight norm as a normalization strategy on conv1d layers.

Reviewed By: geof90

Differential Revision: D16403533

fbshipit-source-id: 0890da42fc0fb5fa80c5985718fa1ae8f51b9b1b

Author: shreydesai

pytext/config/module_config.py

@@ -22,6 +22,8 @@ class CNNParams(ConfigBase):
     kernel_num: int = 100
     # Kernel sizes to use in convolution
     kernel_sizes: List[int] = [3, 4]
+    # Use weight norm in convolution
+    weight_norm: bool = False
 
 
 class PoolingType(Enum):

pytext/models/representations/deepcnn.py

@@ -30,6 +30,7 @@ def __init__(self, config: Config, embed_dim: int) -> None:
 
         out_channels = config.cnn.kernel_num
         kernel_sizes = config.cnn.kernel_sizes
+        weight_norm = config.cnn.weight_norm
 
         conv_layers = []
         linear_layers = []
@@ -46,6 +47,9 @@ def __init__(self, config: Config, embed_dim: int) -> None:
             single_conv = nn.Conv1d(
                 in_channels, 2 * out_channels, k, padding=int((k - 1) / 2)
             )
+            single_conv = (
+                nn.utils.weight_norm(single_conv) if weight_norm else single_conv
+            )
             conv_layers.append(single_conv)
             in_channels = out_channels