Merge branch 'confluence-server-api-time-fix' into confluence-perm-sync

Author: Weves

backend/alembic/versions/7a70b7664e37_add_models_configuration_table.py renamed to backend/alembic/versions/7a70b7664e37_add_model_configuration_table.py

@@ -1,7 +1,7 @@
-"""Add models-configuration table
+"""Add model-configuration table
 
 Revision ID: 7a70b7664e37
-Revises: cf90764725d8
+Revises: d961aca62eb3
 Create Date: 2025-04-10 15:00:35.984669
 
 """
@@ -58,16 +58,13 @@ def upgrade() -> None:
 
     for llm_provider in llm_providers:
         provider_id = llm_provider[0]
-        model_names = llm_provider[1] or []
-        display_model_names = llm_provider[2] or []
-
-        # Create a set of display models for quick lookup
-        display_set = set(display_model_names)
+        model_names_set = set(llm_provider[1] or [])
+        display_names_set = set(llm_provider[2] or [])
 
         # Insert all models from model_names
-        for model_name in model_names:
+        for model_name in model_names_set:
             # If model is in display_model_names, set is_visible to True
-            is_visible = model_name in display_set
+            is_visible = model_name in display_names_set
 
             connection.execute(
                 model_configuration_table.insert().values(

backend/ee/onyx/server/tenants/product_gating.py

@@ -7,7 +7,7 @@
 from onyx.server.settings.models import ApplicationStatus
 from onyx.server.settings.store import load_settings
 from onyx.server.settings.store import store_settings
-from onyx.setup import setup_logger
+from onyx.utils.logger import setup_logger
 from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR
 
 logger = setup_logger()

backend/ee/onyx/server/tenants/user_mapping.py

@@ -9,7 +9,7 @@
 from onyx.db.engine import get_session_with_tenant
 from onyx.db.models import UserTenantMapping
 from onyx.server.manage.models import TenantSnapshot
-from onyx.setup import setup_logger
+from onyx.utils.logger import setup_logger
 from shared_configs.configs import MULTI_TENANT
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR

backend/onyx/background/indexing/job_client.py

@@ -17,7 +17,7 @@
 
 from onyx.configs.constants import POSTGRES_CELERY_WORKER_INDEXING_CHILD_APP_NAME
 from onyx.db.engine import SqlEngine
-from onyx.setup import setup_logger
+from onyx.utils.logger import setup_logger
 from shared_configs.configs import POSTGRES_DEFAULT_SCHEMA
 from shared_configs.configs import TENANT_ID_PREFIX
 from shared_configs.contextvars import CURRENT_TENANT_ID_CONTEXTVAR

backend/onyx/connectors/confluence/connector.py

@@ -1,4 +1,5 @@
 import copy
+import time
 from datetime import datetime
 from datetime import timedelta
 from datetime import timezone
@@ -465,11 +466,13 @@ def _fetch_document_batches(
         # number of documents/errors yielded
         yield_count = 0
 
+        end = (end or time.time()) + 3600 * 24
         checkpoint = copy.deepcopy(checkpoint)
         prev_doc_ids = checkpoint.last_seen_doc_ids
         checkpoint.last_seen_doc_ids = []
         # use "start" when last_updated is 0
-        page_query = self._construct_page_query(checkpoint.last_updated or start, end)
+        start_ts = checkpoint.last_updated or start
+        page_query = self._construct_page_query(start_ts, end)
         logger.debug(f"page_query: {page_query}")
 
         # most requests will include a few pages to skip, so we limit each page to
@@ -480,13 +483,23 @@ def _fetch_document_batches(
             limit=2 * self.batch_size,
         ):
             # create checkpoint after enough documents have been processed
-            if yield_count >= self.batch_size:
-                return checkpoint
+            # if yield_count >= self.batch_size:
+            #     return checkpoint
 
             if page["id"] in prev_doc_ids:
                 # There are a few seconds of fuzziness in the request,
                 # so we skip if we saw this page on the last run
                 continue
+
+            ts = datetime_from_string(page["version"]["when"]).timestamp()
+
+            if ts < checkpoint.last_updated:
+                logger.warning(
+                    f"Confluence Returned results out of order. Request start time: {start_ts}, "
+                    f"current item time: {ts}, checkpoint.last_updated: {checkpoint.last_updated}"
+                )
+                continue
+
             # Build doc from page
             doc_or_failure = self._convert_page_to_document(page)
             yield_count += 1
@@ -495,9 +508,7 @@ def _fetch_document_batches(
                 yield doc_or_failure
                 continue
 
-            checkpoint.last_updated = datetime_from_string(
-                page["version"]["when"]
-            ).timestamp()
+            checkpoint.last_updated = ts
 
             # Now get attachments for that page:
             doc_or_failure = self._fetch_page_attachments(page, doc_or_failure)

backend/onyx/db/sync_record.py

@@ -8,7 +8,7 @@
 from onyx.db.enums import SyncStatus
 from onyx.db.enums import SyncType
 from onyx.db.models import SyncRecord
-from onyx.setup import setup_logger
+from onyx.utils.logger import setup_logger
 
 logger = setup_logger()
 

backend/onyx/server/user_documents/api.py

@@ -45,7 +45,7 @@
 from onyx.server.user_documents.models import MessageResponse
 from onyx.server.user_documents.models import UserFileSnapshot
 from onyx.server.user_documents.models import UserFolderSnapshot
-from onyx.setup import setup_logger
+from onyx.utils.logger import setup_logger
 from shared_configs.contextvars import get_current_tenant_id
 
 logger = setup_logger()