fs: fix the error message when file open flags are a bad type

Before, it would show "TypeError: flags must be an int", which
in native Node is technically correct. But in JS they may be
(and usually are) strings.

Fixes #2871

Author: ronkorving

lib/fs.js

@@ -465,11 +465,16 @@ fs.readFileSync = function(path, options) {
 
 // Used by binding.open and friends
 function stringToFlags(flag) {
-  // Only mess with strings
-  if (typeof flag !== 'string') {
+  // Integers are already good to go
+  if (Number.isInteger(flag)) {
     return flag;
   }
 
+  // We only allow strings from here on
+  if (typeof flag !== 'string') {
+    throw new TypeError('File open flags may only be string or integer');
+  }
+
   switch (flag) {
     case 'r' : return O_RDONLY;
     case 'rs' : // fall through

test/parallel/test-fs-open.js

@@ -32,8 +32,15 @@ fs.open(__filename, 'rs', function(err, fd) {
   openFd2 = fd;
 });
 
+assert.throws(function() {
+  fs.open('/path/to/file/that/does/not/exist', { bad: 'flags' }, assert.fail);
+}, function(err) {
+  if (err instanceof TypeError) {
+    return err.message === 'File open flags may only be string or integer';
+  }
+});
+
 process.on('exit', function() {
   assert.ok(openFd);
   assert.ok(openFd2);
 });
-