Restrict to server only

Author: Weves

backend/onyx/background/indexing/run_indexing.py

@@ -15,7 +15,6 @@
 from onyx.configs.app_configs import INDEX_BATCH_SIZE
 from onyx.configs.app_configs import INDEXING_SIZE_WARNING_THRESHOLD
 from onyx.configs.app_configs import INDEXING_TRACER_INTERVAL
-from onyx.configs.app_configs import INTEGRATION_TESTS_MODE
 from onyx.configs.app_configs import LEAVE_CONNECTOR_ACTIVE_ON_INITIALIZATION_FAILURE
 from onyx.configs.app_configs import POLL_CONNECTOR_OFFSET
 from onyx.configs.constants import DocumentSource
@@ -97,8 +96,8 @@ def _get_connector_runner(
         )
 
         # validate the connector settings
-        if not INTEGRATION_TESTS_MODE:
-            runnable_connector.validate_connector_settings()
+        # if not INTEGRATION_TESTS_MODE:
+        # runnable_connector.validate_connector_settings()
 
     except UnexpectedValidationError as e:
         logger.exception(

backend/onyx/connectors/confluence/connector.py

@@ -5,7 +5,6 @@
 from typing import Any
 from urllib.parse import quote
 
-from requests.exceptions import HTTPError
 from typing_extensions import override
 
 from onyx.configs.app_configs import CONFLUENCE_CONNECTOR_LABELS_TO_SKIP
@@ -22,10 +21,6 @@
 from onyx.connectors.confluence.utils import process_attachment
 from onyx.connectors.confluence.utils import update_param_in_path
 from onyx.connectors.confluence.utils import validate_attachment_filetype
-from onyx.connectors.exceptions import ConnectorValidationError
-from onyx.connectors.exceptions import CredentialExpiredError
-from onyx.connectors.exceptions import InsufficientPermissionsError
-from onyx.connectors.exceptions import UnexpectedValidationError
 from onyx.connectors.interfaces import CheckpointedConnector
 from onyx.connectors.interfaces import CheckpointOutput
 from onyx.connectors.interfaces import ConnectorCheckpoint
@@ -645,28 +640,30 @@ def retrieve_all_slim_documents(
         yield doc_metadata_list
 
     def validate_connector_settings(self) -> None:
-        try:
-            spaces = self.low_timeout_confluence_client.get_all_spaces(limit=1)
-        except HTTPError as e:
-            status_code = e.response.status_code if e.response else None
-            if status_code == 401:
-                raise CredentialExpiredError(
-                    "Invalid or expired Confluence credentials (HTTP 401)."
-                )
-            elif status_code == 403:
-                raise InsufficientPermissionsError(
-                    "Insufficient permissions to access Confluence resources (HTTP 403)."
-                )
-            raise UnexpectedValidationError(
-                f"Unexpected Confluence error (status={status_code}): {e}"
-            )
-        except Exception as e:
-            raise UnexpectedValidationError(
-                f"Unexpected error while validating Confluence settings: {e}"
-            )
-
-        if not spaces or not spaces.get("results"):
-            raise ConnectorValidationError(
-                "No Confluence spaces found. Either your credentials lack permissions, or "
-                "there truly are no spaces in this Confluence instance."
-            )
+        # Freezing often, just removing for now
+        return None
+        # try:
+        #     spaces = self.low_timeout_confluence_client.get_all_spaces(limit=1)
+        # except HTTPError as e:
+        #     status_code = e.response.status_code if e.response else None
+        #     if status_code == 401:
+        #         raise CredentialExpiredError(
+        #             "Invalid or expired Confluence credentials (HTTP 401)."
+        #         )
+        #     elif status_code == 403:
+        #         raise InsufficientPermissionsError(
+        #             "Insufficient permissions to access Confluence resources (HTTP 403)."
+        #         )
+        #     raise UnexpectedValidationError(
+        #         f"Unexpected Confluence error (status={status_code}): {e}"
+        #     )
+        # except Exception as e:
+        #     raise UnexpectedValidationError(
+        #         f"Unexpected error while validating Confluence settings: {e}"
+        #     )
+
+        # if not spaces or not spaces.get("results"):
+        #     raise ConnectorValidationError(
+        #         "No Confluence spaces found. Either your credentials lack permissions, or "
+        #         "there truly are no spaces in this Confluence instance."
+        #     )

backend/onyx/connectors/confluence/onyx_confluence.py

@@ -465,7 +465,9 @@ def _paginate_url(
                     )
                     continue
 
-                if raw_response.status_code == 500:
+                # this iterative approach only works for server, since cloud uses cursor-based
+                # pagination
+                if raw_response.status_code == 500 and not self._is_cloud:
                     # in the case of a 500 error, go one by one until we find the error
                     # and skip the error
                     results: list[dict[str, Any]] = []
@@ -480,6 +482,7 @@ def _paginate_url(
 
                     found_empty_page = False
                     temp_url_suffix = url_suffix
+                    ind = 0
                     for ind in range(limit):
                         try:
                             temp_url_suffix = update_param_in_path(
@@ -510,14 +513,17 @@ def _paginate_url(
                             )
 
                     if found_empty_page:
+                        if next_page_callback:
+                            next_page_callback("")
                         break
 
                     # since we manually tried up to limit, we can just update the start
                     url_suffix = update_param_in_path(
-                        url_suffix, "start", str(initial_start + limit)
+                        url_suffix, "start", str(initial_start + ind + 1)
                     )
                     if next_page_callback:
                         next_page_callback(url_suffix)
+
                     continue
 
                 else: