move test utilies into scrapy_poet/utils/

Author: BurnzZ

scrapy_poet/utils.py

@@ -1,3 +1,4 @@
+import os
 from inspect import isasyncgenfunction
 from typing import Dict
 from unittest import mock
@@ -6,30 +7,57 @@
 from scrapy import signals
 from scrapy.crawler import Crawler
 from scrapy.exceptions import CloseSpider
+from scrapy.http import Request, Response
 from scrapy.settings import Settings
+from scrapy.utils.project import inside_project, project_data_dir
 from scrapy.utils.python import to_bytes
 from twisted.internet import reactor
 from twisted.internet.task import deferLater
 from twisted.web.resource import Resource
 from twisted.web.server import NOT_DONE_YET
+from web_poet import HttpRequest, HttpResponse, HttpResponseHeaders
 
-from tests.mockserver import MockServer
+from scrapy_poet.utils.mockserver import MockServer
 
 
-def create_scrapy_settings(request):
-    """Default scrapy-poet settings"""
-    s = dict(
-        # collect scraped items to crawler.spider.collected_items
-        ITEM_PIPELINES={
-            "tests.utils.CollectorPipeline": 100,
-        },
-        DOWNLOADER_MIDDLEWARES={
-            # collect injected dependencies to crawler.spider.collected_response_deps
-            "tests.utils.InjectedDependenciesCollectorMiddleware": 542,
-            "scrapy_poet.InjectionMiddleware": 543,
-        },
+def get_scrapy_data_path(createdir: bool = True, default_dir: str = ".scrapy") -> str:
+    """Return a path to a folder where Scrapy is storing data.
+
+    Usually that's a .scrapy folder inside the project.
+    """
+    # This code is extracted from scrapy.utils.project.data_path function,
+    # which does too many things.
+    path = project_data_dir() if inside_project() else default_dir
+    if createdir:
+        os.makedirs(path, exist_ok=True)
+    return path
+
+
+def http_request_to_scrapy_request(request: HttpRequest, **kwargs) -> Request:
+    return Request(
+        url=str(request.url),
+        method=request.method,
+        headers=request.headers,
+        body=request.body,
+        **kwargs,
+    )
+
+
+def scrapy_response_to_http_response(response: Response) -> HttpResponse:
+    """Convenience method to convert a ``scrapy.http.Response`` into a
+    ``web_poet.HttpResponse``.
+    """
+    kwargs = {}
+    encoding = getattr(response, "_encoding", None)
+    if encoding:
+        kwargs["encoding"] = encoding
+    return HttpResponse(
+        url=response.url,
+        body=response.body,
+        status=response.status,
+        headers=HttpResponseHeaders.from_bytes_dict(response.headers),
+        **kwargs,
     )
-    return Settings(s)
 
 
 class HtmlResource(Resource):
@@ -161,6 +189,22 @@ def process_response(self, request, response, spider):
         return response
 
 
+def create_scrapy_settings(request):
+    """Default scrapy-poet settings"""
+    s = dict(
+        # collect scraped items to crawler.spider.collected_items
+        ITEM_PIPELINES={
+            CollectorPipeline: 100,
+        },
+        DOWNLOADER_MIDDLEWARES={
+            # collect injected dependencies to crawler.spider.collected_response_deps
+            InjectedDependenciesCollectorMiddleware: 542,
+            "scrapy_poet.InjectionMiddleware": 543,
+        },
+    )
+    return Settings(s)
+
+
 def capture_exceptions(callback):
     """Wrapper for Scrapy callbacks that captures exceptions within
     the provided callback and yields it under `exception` property. Also

tests/utils.py renamed to scrapy_poet/utils/__init__.py

@@ -29,7 +29,7 @@ def __enter__(self):
                 sys.executable,
                 "-u",
                 "-m",
-                "tests.mockserver",
+                "scrapy_poet.utils.mockserver",
                 self.resource,
                 "--port",
                 str(self.port),

tests/mockserver.py renamed to scrapy_poet/utils/mockserver.py

@@ -1,6 +1,6 @@
 import pytest
 
-from tests.utils import create_scrapy_settings
+from scrapy_poet.utils import create_scrapy_settings
 
 
 @pytest.fixture()

tests/conftest.py

@@ -15,15 +15,15 @@
 from web_poet.pages import WebPage
 
 from scrapy_poet.downloader import create_scrapy_downloader
-from scrapy_poet.utils import http_request_to_scrapy_request
-from tests.utils import (
+from scrapy_poet.utils import (
     AsyncMock,
     DelayedResource,
     EchoResource,
-    MockServer,
     StatusResource,
+    http_request_to_scrapy_request,
     make_crawler,
 )
+from scrapy_poet.utils.mockserver import MockServer
 
 
 @pytest.fixture

tests/test_downloader.py

@@ -17,8 +17,13 @@
 from scrapy_poet import DummyResponse, InjectionMiddleware, callback_for
 from scrapy_poet.cache import SqlitedictCache
 from scrapy_poet.page_input_providers import PageObjectInputProvider
-from tests.mockserver import get_ephemeral_port
-from tests.utils import HtmlResource, capture_exceptions, crawl_items, crawl_single_item
+from scrapy_poet.utils import (
+    HtmlResource,
+    capture_exceptions,
+    crawl_items,
+    crawl_single_item,
+)
+from scrapy_poet.utils.mockserver import get_ephemeral_port
 
 
 class ProductHtml(HtmlResource):

tests/test_middleware.py

@@ -19,7 +19,7 @@
     PageObjectInputProvider,
     PageParamsProvider,
 )
-from tests.utils import AsyncMock, HtmlResource, crawl_single_item
+from scrapy_poet.utils import AsyncMock, HtmlResource, crawl_single_item
 
 
 class ProductHtml(HtmlResource):

tests/test_providers.py

@@ -5,7 +5,7 @@
 from web_poet.exceptions import Retry
 from web_poet.pages import WebPage
 
-from tests.utils import EchoResource, MockServer, make_crawler
+from scrapy_poet.utils import EchoResource, MockServer, make_crawler
 
 
 class BaseSpider(Spider):

tests/test_retries.py

@@ -10,7 +10,7 @@
     HttpResponseProvider,
     PageObjectInputProvider,
 )
-from tests.utils import HtmlResource, crawl_items, crawl_single_item
+from scrapy_poet.utils import HtmlResource, crawl_items, crawl_single_item
 
 
 class ProductHtml(HtmlResource):

tests/test_scrapy_dependencies.py

@@ -28,9 +28,13 @@
 
 from scrapy_poet import callback_for
 from scrapy_poet.downloadermiddlewares import DEFAULT_PROVIDERS
-from tests.mockserver import get_ephemeral_port
+from scrapy_poet.utils import (
+    capture_exceptions,
+    crawl_single_item,
+    create_scrapy_settings,
+)
+from scrapy_poet.utils.mockserver import get_ephemeral_port
 from tests.test_middleware import ProductHtml
-from tests.utils import capture_exceptions, crawl_single_item, create_scrapy_settings
 
 DOMAIN = get_domain(socket.gethostbyname(socket.gethostname()))
 PORT = get_ephemeral_port()