Unit tests fail on master

[tseaver]

From this Kokoro job, for PR #115 which doesn't touch code at all.

$ .nox/unit-3-6/bin/py.test tests/unit/
============================= test session starts ==============================
platform linux -- Python 3.6.10, pytest-6.2.4, py-1.10.0, pluggy-0.13.1
rootdir: /home/tseaver/projects/agendaless/Google/src/python-cloud-core
plugins: cov-2.12.1
collected 168 items                                                            

tests/unit/test__helpers.py ............................................ [ 26%]
...................                                                      [ 37%]
tests/unit/test__http.py ..................F..FFF.FF...........          [ 60%]
tests/unit/test_client.py ..................................             [ 80%]
tests/unit/test_exceptions.py .........                                  [ 85%]
tests/unit/test_obsolete.py ..                                           [ 86%]
tests/unit/test_operation.py ......................                      [100%]

=================================== FAILURES ===================================
_________________ TestJSONConnection.test_api_request_defaults _________________

self = <tests.unit.test__http.TestJSONConnection testMethod=test_api_request_defaults>

    def test_api_request_defaults(self):
        from google.cloud._http import CLIENT_INFO_HEADER
    
        http = make_requests_session(
            [make_response(content=b"{}", headers=self.JSON_HEADERS)]
        )
        client = mock.Mock(_http=http, spec=["_http"])
        conn = self._make_mock_one(client)
        path = "/path/required"
    
>       self.assertEqual(conn.api_request("GET", path), {})

tests/unit/test__http.py:393: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/_http.py:487: in api_request
    return response.json()
.nox/unit-3-6/lib/python3.6/site-packages/requests/models.py:910: in json
    return complexjson.loads(self.text, **kwargs)
/opt/Python-3.6.10/lib/python3.6/json/__init__.py:354: in loads
    return _default_decoder.decode(s)
/opt/Python-3.6.10/lib/python3.6/json/decoder.py:339: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <json.decoder.JSONDecoder object at 0x7f697cdbd828>, s = '筽', idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
        a JSON document) and return a 2-tuple of the Python
        representation and the index in ``s`` where the document ended.
    
        This can be used to decode a JSON document from a string that may
        have extraneous data at the end.
    
        """
        try:
            obj, end = self.scan_once(s, idx)
        except StopIteration as err:
>           raise JSONDecodeError("Expecting value", s, err.value) from None
E           json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

/opt/Python-3.6.10/lib/python3.6/json/decoder.py:357: JSONDecodeError
__________________ TestJSONConnection.test_api_request_w_data __________________

self = <tests.unit.test__http.TestJSONConnection testMethod=test_api_request_w_data>

    def test_api_request_w_data(self):
        from google.cloud._http import CLIENT_INFO_HEADER
    
        http = make_requests_session([self.EMPTY_JSON_RESPONSE])
        client = mock.Mock(_http=http, spec=["_http"])
        conn = self._make_mock_one(client)
    
        data = {"foo": "bar"}
>       self.assertEqual(conn.api_request("POST", "/", data=data), {})

tests/unit/test__http.py:526: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/_http.py:487: in api_request
    return response.json()
.nox/unit-3-6/lib/python3.6/site-packages/requests/models.py:910: in json
    return complexjson.loads(self.text, **kwargs)
/opt/Python-3.6.10/lib/python3.6/json/__init__.py:354: in loads
    return _default_decoder.decode(s)
/opt/Python-3.6.10/lib/python3.6/json/decoder.py:339: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <json.decoder.JSONDecoder object at 0x7f697cdbd828>, s = '筽', idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
        a JSON document) and return a 2-tuple of the Python
        representation and the index in ``s`` where the document ended.
    
        This can be used to decode a JSON document from a string that may
        have extraneous data at the end.
    
        """
        try:
            obj, end = self.scan_once(s, idx)
        except StopIteration as err:
>           raise JSONDecodeError("Expecting value", s, err.value) from None
E           json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

/opt/Python-3.6.10/lib/python3.6/json/decoder.py:357: JSONDecodeError
_____________ TestJSONConnection.test_api_request_w_extra_headers ______________

self = <tests.unit.test__http.TestJSONConnection testMethod=test_api_request_w_extra_headers>

    def test_api_request_w_extra_headers(self):
        from google.cloud._http import CLIENT_INFO_HEADER
    
        http = make_requests_session([self.EMPTY_JSON_RESPONSE])
        client = mock.Mock(_http=http, spec=["_http"])
        conn = self._make_mock_one(client)
        conn.extra_headers = {
            "X-Baz": "dax-quux",
            "X-Foo": "not-bar",  # Collision with ``headers``.
        }
    
>       result = conn.api_request("GET", "/", headers={"X-Foo": "bar"})

tests/unit/test__http.py:499: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/_http.py:487: in api_request
    return response.json()
.nox/unit-3-6/lib/python3.6/site-packages/requests/models.py:910: in json
    return complexjson.loads(self.text, **kwargs)
/opt/Python-3.6.10/lib/python3.6/json/__init__.py:354: in loads
    return _default_decoder.decode(s)
/opt/Python-3.6.10/lib/python3.6/json/decoder.py:339: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <json.decoder.JSONDecoder object at 0x7f697cdbd828>, s = '筽', idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
        a JSON document) and return a 2-tuple of the Python
        representation and the index in ``s`` where the document ended.
    
        This can be used to decode a JSON document from a string that may
        have extraneous data at the end.
    
        """
        try:
            obj, end = self.scan_once(s, idx)
        except StopIteration as err:
>           raise JSONDecodeError("Expecting value", s, err.value) from None
E           json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

/opt/Python-3.6.10/lib/python3.6/json/decoder.py:357: JSONDecodeError
________________ TestJSONConnection.test_api_request_w_headers _________________

self = <tests.unit.test__http.TestJSONConnection testMethod=test_api_request_w_headers>

    def test_api_request_w_headers(self):
        from google.cloud._http import CLIENT_INFO_HEADER
    
        http = make_requests_session([self.EMPTY_JSON_RESPONSE])
        client = mock.Mock(_http=http, spec=["_http"])
        conn = self._make_mock_one(client)
    
>       result = conn.api_request("GET", "/", headers={"X-Foo": "bar"})

tests/unit/test__http.py:471: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/_http.py:487: in api_request
    return response.json()
.nox/unit-3-6/lib/python3.6/site-packages/requests/models.py:910: in json
    return complexjson.loads(self.text, **kwargs)
/opt/Python-3.6.10/lib/python3.6/json/__init__.py:354: in loads
    return _default_decoder.decode(s)
/opt/Python-3.6.10/lib/python3.6/json/decoder.py:339: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <json.decoder.JSONDecoder object at 0x7f697cdbd828>, s = '筽', idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
        a JSON document) and return a 2-tuple of the Python
        representation and the index in ``s`` where the document ended.
    
        This can be used to decode a JSON document from a string that may
        have extraneous data at the end.
    
        """
        try:
            obj, end = self.scan_once(s, idx)
        except StopIteration as err:
>           raise JSONDecodeError("Expecting value", s, err.value) from None
E           json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

/opt/Python-3.6.10/lib/python3.6/json/decoder.py:357: JSONDecodeError
______________ TestJSONConnection.test_api_request_w_query_params ______________

self = <tests.unit.test__http.TestJSONConnection testMethod=test_api_request_w_query_params>

    def test_api_request_w_query_params(self):
        from six.moves.urllib.parse import parse_qs
        from six.moves.urllib.parse import urlsplit
        from google.cloud._http import CLIENT_INFO_HEADER
    
        http = make_requests_session([self.EMPTY_JSON_RESPONSE])
        client = mock.Mock(_http=http, spec=["_http"])
        conn = self._make_mock_one(client)
    
>       result = conn.api_request("GET", "/", {"foo": "bar", "baz": ["qux", "quux"]})

tests/unit/test__http.py:437: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/_http.py:487: in api_request
    return response.json()
.nox/unit-3-6/lib/python3.6/site-packages/requests/models.py:910: in json
    return complexjson.loads(self.text, **kwargs)
/opt/Python-3.6.10/lib/python3.6/json/__init__.py:354: in loads
    return _default_decoder.decode(s)
/opt/Python-3.6.10/lib/python3.6/json/decoder.py:339: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <json.decoder.JSONDecoder object at 0x7f697cdbd828>, s = '筽', idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
        a JSON document) and return a 2-tuple of the Python
        representation and the index in ``s`` where the document ended.
    
        This can be used to decode a JSON document from a string that may
        have extraneous data at the end.
    
        """
        try:
            obj, end = self.scan_once(s, idx)
        except StopIteration as err:
>           raise JSONDecodeError("Expecting value", s, err.value) from None
E           json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

/opt/Python-3.6.10/lib/python3.6/json/decoder.py:357: JSONDecodeError
________________ TestJSONConnection.test_api_request_w_timeout _________________

self = <tests.unit.test__http.TestJSONConnection testMethod=test_api_request_w_timeout>

    def test_api_request_w_timeout(self):
        from google.cloud._http import CLIENT_INFO_HEADER
    
        http = make_requests_session(
            [make_response(content=b"{}", headers=self.JSON_HEADERS)]
        )
        client = mock.Mock(_http=http, spec=["_http"])
        conn = self._make_mock_one(client)
        path = "/path/required"
    
>       self.assertEqual(conn.api_request("GET", path, timeout=(2.2, 3.3)), {})

tests/unit/test__http.py:555: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
google/cloud/_http.py:487: in api_request
    return response.json()
.nox/unit-3-6/lib/python3.6/site-packages/requests/models.py:910: in json
    return complexjson.loads(self.text, **kwargs)
/opt/Python-3.6.10/lib/python3.6/json/__init__.py:354: in loads
    return _default_decoder.decode(s)
/opt/Python-3.6.10/lib/python3.6/json/decoder.py:339: in decode
    obj, end = self.raw_decode(s, idx=_w(s, 0).end())
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = <json.decoder.JSONDecoder object at 0x7f697cdbd828>, s = '筽', idx = 0

    def raw_decode(self, s, idx=0):
        """Decode a JSON document from ``s`` (a ``str`` beginning with
        a JSON document) and return a 2-tuple of the Python
        representation and the index in ``s`` where the document ended.
    
        This can be used to decode a JSON document from a string that may
        have extraneous data at the end.
    
        """
        try:
            obj, end = self.scan_once(s, idx)
        except StopIteration as err:
>           raise JSONDecodeError("Expecting value", s, err.value) from None
E           json.decoder.JSONDecodeError: Expecting value: line 1 column 1 (char 0)

/opt/Python-3.6.10/lib/python3.6/json/decoder.py:357: JSONDecodeError
=============================== warnings summary ===============================
tests/unit/test__http.py::TestJSONConnection::test_api_request_defaults
tests/unit/test__http.py::TestJSONConnection::test_api_request_w_data
tests/unit/test__http.py::TestJSONConnection::test_api_request_w_extra_headers
tests/unit/test__http.py::TestJSONConnection::test_api_request_w_headers
tests/unit/test__http.py::TestJSONConnection::test_api_request_w_query_params
tests/unit/test__http.py::TestJSONConnection::test_api_request_w_timeout
  /home/tseaver/projects/agendaless/Google/src/python-cloud-core/.nox/unit-3-6/lib/python3.6/site-packages/charset_normalizer/api.py:95: UserWarning: Trying to detect encoding from a tiny portion of (2) byte(s).
    warn('Trying to detect encoding from a tiny portion of ({}) byte(s).'.format(length))

tests/unit/test_exceptions.py::test_from_http_response_text_content
  /home/tseaver/projects/agendaless/Google/src/python-cloud-core/.nox/unit-3-6/lib/python3.6/site-packages/charset_normalizer/api.py:95: UserWarning: Trying to detect encoding from a tiny portion of (7) byte(s).
    warn('Trying to detect encoding from a tiny portion of ({}) byte(s).'.format(length))

-- Docs: https://docs.pytest.org/en/stable/warnings.html
=========================== short test summary info ============================
FAILED tests/unit/test__http.py::TestJSONConnection::test_api_request_defaults
FAILED tests/unit/test__http.py::TestJSONConnection::test_api_request_w_data
FAILED tests/unit/test__http.py::TestJSONConnection::test_api_request_w_extra_headers
FAILED tests/unit/test__http.py::TestJSONConnection::test_api_request_w_headers
FAILED tests/unit/test__http.py::TestJSONConnection::test_api_request_w_query_params
FAILED tests/unit/test__http.py::TestJSONConnection::test_api_request_w_timeout
================== 6 failed, 162 passed, 7 warnings in 1.30s ===================

[tseaver]

The failures are due to the release yesterday of requests 2.26.0. psf/requests#5797 is the PR which made this change:

Instead of chardet, use the MIT-licensed charset_normalizer for Python3 to remove license ambiguity for projects bundling requests. If chardet is already installed on your machine it will be used instead of charset_normalizer to keep backwards compatibility. (#5797)

You can also install chardet while installing requests by specifying [use_chardet_on_py3] extra as follows:

    `shell pip install "requests[use_chardet_on_py3]" `

Python2 still depends upon the chardet module.

[tseaver]

I have verified that forcibly installing chardet fixes the failures.

[tseaver]

The fix I chose for #118 was to add an explicit encoding to the response, rather than inject more local noise into noxfile.py.