🐛 [Bug] RuntimeError: required keyword attribute 'to' is undefined

[bowang007]

Bug Description

Run into this error when using Python APIs:

DEBUG: [Torch-TensorRT TorchScript Conversion Context] - Evaluating %3 : int = prim::Constant[value=1]()
Traceback (most recent call last):
  File "add.py", line 41, in <module>
    main()
  File "add.py", line 27, in main
    trt_mod = torch_tensorrt.compile(scripted_model, **compile_settings)
  File "/home/bo/.virtualenvs/my_env/lib/python3.8/site-packages/torch_tensorrt/_compile.py", line 125, in compile
    return torch_tensorrt.ts.compile(
  File "/home/bo/.virtualenvs/my_env/lib/python3.8/site-packages/torch_tensorrt/ts/_compiler.py", line 136, in compile
    compiled_cpp_mod = _C.compile_graph(module._c, _parse_compile_spec(spec))
RuntimeError: required keyword attribute 'to' is undefined

To Reproduce

Every model I tested failed, even for some very small models like :

class Add(torch.nn.Module):

    def __init__(self):
        super(Add, self).__init__()

    def forward(self, x, y):
        return x + y


def main():

    model = Add().eval().cuda()
    scripted_model = torch.jit.script(model)
    torch.jit.save(scripted_model, "add.jit")
    compile_settings = {
        "inputs": [torch_tensorrt.Input([1, 3, 5, 5]), torch_tensorrt.Input([1, 3, 5, 5])],
        "enabled_precisions": {torch.float32},
    }

    trt_mod = torch_tensorrt.compile(scripted_model, **compile_settings)
    # import pdb; pdb.set_trace()
    print("done")
    input_tensor1 = torch.randint(0, 5, (1, 3, 5, 5), dtype=torch.float32).cuda()
    input_tensor2 = torch.randint(0, 5, (1, 3, 5, 5), dtype=torch.float32).cuda()

    norm_trt_ts = torch_tensorrt.compile(model, compile_settings)
    trt_outputs = norm_trt_ts.forward(input_tensor1, input_tensor2)