Support bytes input in joint intent-slot model OSS (#745)

Summary:
Pull Request resolved: #745

As title

Differential Revision: D16078832

fbshipit-source-id: e10d4cb5c01ae7ba71d72ba0edc0af54ac0190db

Author: geof90

pytext/models/word_model.py

@@ -3,10 +3,15 @@
 
 from typing import Union
 
-from pytext.data.tensorizers import SlotLabelTensorizer, TokenTensorizer
+from pytext.data.tensorizers import (
+    ByteTokenTensorizer,
+    SlotLabelTensorizer,
+    TokenTensorizer,
+)
 from pytext.data.utils import UNK
+from pytext.exporters.exporter import ModelExporter
 from pytext.models.decoders.mlp_decoder import MLPDecoder
-from pytext.models.embeddings import WordEmbedding
+from pytext.models.embeddings import CharacterEmbedding, WordEmbedding
 from pytext.models.model import Model
 from pytext.models.module import create_module
 from pytext.models.output_layers import CRFOutputLayer, WordTaggingOutputLayer
@@ -54,11 +59,17 @@ def __init__(self, *args, **kwargs):
 class WordTaggingModel(Model):
     class Config(Model.Config):
         class ModelInput(Model.Config.ModelInput):
-            tokens: TokenTensorizer.Config = TokenTensorizer.Config()
+            # We should support characters as well, but CharacterTokenTensorizer
+            # does not support adding characters to vocab yet.
+            tokens: Union[
+                ByteTokenTensorizer.Config, TokenTensorizer.Config
+            ] = TokenTensorizer.Config()
             labels: SlotLabelTensorizer.Config = SlotLabelTensorizer.Config()
 
         inputs: ModelInput = ModelInput()
-        embedding: WordEmbedding.Config = WordEmbedding.Config()
+        embedding: Union[
+            WordEmbedding.Config, CharacterEmbedding.Config
+        ] = WordEmbedding.Config()
 
         representation: Union[
             BiLSTMSlotAttention.Config,  # TODO: make default when sorting solved
@@ -72,10 +83,21 @@ class ModelInput(Model.Config.ModelInput):
 
     @classmethod
     def create_embedding(cls, config, tensorizers):
-        vocab = tensorizers["tokens"].vocab
-        return WordEmbedding(
-            len(vocab), config.embedding.embed_dim, None, None, vocab.idx[UNK], []
-        )
+        token_tensorizer = tensorizers["tokens"]
+        if isinstance(token_tensorizer, TokenTensorizer):
+            vocab = token_tensorizer.vocab
+            return WordEmbedding(
+                len(vocab), config.embedding.embed_dim, None, None, vocab.idx[UNK], []
+            )
+        else:
+            return CharacterEmbedding(
+                token_tensorizer.NUM_BYTES,
+                config.embedding.embed_dim,
+                config.embedding.cnn.kernel_num,
+                config.embedding.cnn.kernel_sizes,
+                config.embedding.highway_layers,
+                config.embedding.projection_dim,
+            )
 
     @classmethod
     def from_config(cls, config, tensorizers):
@@ -108,3 +130,26 @@ def arrange_model_inputs(self, tensor_dict):
 
     def arrange_targets(self, tensor_dict):
         return tensor_dict["labels"]
+
+    def get_export_input_names(self, tensorizers):
+        return "tokens", "tokens_lens"
+
+    def get_export_output_names(self, tensorizers):
+        return ["word_scores"]
+
+    def vocab_to_export(self, tensorizers):
+        token_tensorizer = tensorizers["tokens"]
+        if isinstance(token_tensorizer, TokenTensorizer):
+            return {"tokens": list(token_tensorizer.vocab)}
+
+        return {}
+
+    def caffe2_export(self, tensorizers, tensor_dict, path, export_onnx_path=None):
+        exporter = ModelExporter(
+            ModelExporter.Config(),
+            self.get_export_input_names(tensorizers),
+            self.arrange_model_inputs(tensor_dict),
+            self.vocab_to_export(tensorizers),
+            self.get_export_output_names(tensorizers),
+        )
+        return exporter.export_to_caffe2(self, path, export_onnx_path=export_onnx_path)