source-microsoft-sharepoint: add cretaed_at

Author: aldogonzalez8

airbyte-integrations/connectors/source-microsoft-sharepoint/poetry.lock

@@ -20,7 +20,7 @@ python = "^3.11,<3.12"
 msal = "==1.27.0"
 Office365-REST-Python-Client = "==2.5.5"
 smart-open = "==6.4.0"
-airbyte-cdk = {extras = ["file-based"], version = "6.45.4.post48.dev14477787653"}
+airbyte-cdk = {extras = ["file-based"], version = "6.45.4.post49.dev14495925594"}
 
 [tool.poetry.scripts]
 source-microsoft-sharepoint = "source_microsoft_sharepoint.run:run"

airbyte-integrations/connectors/source-microsoft-sharepoint/pyproject.toml

@@ -8,7 +8,6 @@
 from datetime import datetime
 from functools import lru_cache
 from io import IOBase
-from os import makedirs, path
 from os.path import getsize
 from typing import Any, Callable, Dict, Iterable, List, MutableMapping, Optional, Tuple
 
@@ -30,7 +29,7 @@
 from airbyte_cdk.sources.file_based.remote_file import RemoteFile
 from source_microsoft_sharepoint.spec import SourceMicrosoftSharePointSpec
 
-from .exceptions import ErrorDownloadingFile, ErrorFetchingMetadata
+from .exceptions import ErrorFetchingMetadata
 from .utils import (
     FolderNotFoundException,
     MicrosoftSharePointRemoteFile,
@@ -167,7 +166,7 @@ def config(self, value: SourceMicrosoftSharePointSpec):
         assert isinstance(value, SourceMicrosoftSharePointSpec)
         self._config = value
 
-    def _get_shared_drive_object(self, drive_id: str, object_id: str, path: str) -> List[Tuple[str, str, datetime]]:
+    def _get_shared_drive_object(self, drive_id: str, object_id: str, path: str) -> Iterable[MicrosoftSharePointRemoteFile]:
         """
         Retrieves a list of all nested files under the specified object.
 
@@ -176,7 +175,7 @@ def _get_shared_drive_object(self, drive_id: str, object_id: str, path: str) ->
             object_id: The ID of the object to start the search from.
 
         Returns:
-            A list of tuples containing file information (name, download URL, and last modified datetime).
+            An iterable of MicrosoftSharePointRemoteFile instances containing file information.
 
         Raises:
             RuntimeError: If an error occurs during the request.
@@ -186,7 +185,7 @@ def _get_shared_drive_object(self, drive_id: str, object_id: str, path: str) ->
         headers = {"Authorization": f"Bearer {access_token}"}
         base_url = f"https://graph.microsoft.com/v1.0/drives/{drive_id}"
 
-        def get_files(url: str, path: str) -> List[Tuple[str, str, datetime]]:
+        def get_files(url: str, path: str) -> Iterable[MicrosoftSharePointRemoteFile]:
             response = requests.get(url, headers=headers)
             if response.status_code != 200:
                 error_info = response.json().get("error", {}).get("message", "No additional error information provided.")
@@ -196,8 +195,15 @@ def get_files(url: str, path: str) -> List[Tuple[str, str, datetime]]:
             for child in data.get("value", []):
                 new_path = path + "/" + child["name"]
                 if child.get("file"):  # Object is a file
+                    # last_modified and created_at are type string e.g. "2025-04-16T14:41:00Z"
                     last_modified = datetime.strptime(child["lastModifiedDateTime"], "%Y-%m-%dT%H:%M:%SZ")
-                    yield (new_path, child["@microsoft.graph.downloadUrl"], last_modified)
+                    created_at = datetime.strptime(child["createdDateTime"], "%Y-%m-%dT%H:%M:%SZ")
+                    yield MicrosoftSharePointRemoteFile(
+                        uri=new_path,
+                        download_url=child["@microsoft.graph.downloadUrl"],
+                        last_modified=last_modified,
+                        created_at=created_at
+                    )
                 else:  # Object is a folder, retrieve children
                     child_url = f"{base_url}/items/{child['id']}/children"  # Use item endpoint for nested objects
                     yield from get_files(child_url, new_path)
@@ -218,23 +224,35 @@ def get_files(url: str, path: str) -> List[Tuple[str, str, datetime]]:
         if item_data.get("file"):  # Initial object is a file
             new_path = path + "/" + item_data["name"]
             last_modified = datetime.strptime(item_data["lastModifiedDateTime"], "%Y-%m-%dT%H:%M:%SZ")
-            yield (new_path, item_data["@microsoft.graph.downloadUrl"], last_modified)
+            created_at = datetime.strptime(item_data["createdDateTime"], "%Y-%m-%dT%H:%M:%SZ")
+            yield MicrosoftSharePointRemoteFile(
+                uri=new_path,
+                download_url=item_data["@microsoft.graph.downloadUrl"],
+                last_modified=last_modified,
+                created_at=created_at
+            )
         else:
             # Initial object is a folder, start file retrieval
             yield from get_files(f"{item_url}/children", path)
 
-    def _list_directories_and_files(self, root_folder, path):
+    def _list_directories_and_files(self, root_folder, path) -> Iterable[MicrosoftSharePointRemoteFile]:
         """Enumerates folders and files starting from a root folder."""
         drive_items = execute_query_with_retry(root_folder.children.get())
         for item in drive_items:
             item_path = path + "/" + item.name if path else item.name
             if item.is_file:
-                yield (item_path, item.properties["@microsoft.graph.downloadUrl"], item.properties["lastModifiedDateTime"])
+                # last_modified and created_at are type datetime.datetime e.g. (2025, 2, 18, 19, 32, 4)
+                yield MicrosoftSharePointRemoteFile(
+                    uri=item_path,
+                    download_url=item.properties["@microsoft.graph.downloadUrl"],
+                    last_modified=item.properties["lastModifiedDateTime"],
+                    created_at=item.properties["createdDateTime"]
+                )
             else:
                 yield from self._list_directories_and_files(item, item_path)
         yield from []
 
-    def _get_files_by_drive_name(self, drives, folder_path):
+    def _get_files_by_drive_name(self, drives, folder_path) -> Iterable[MicrosoftSharePointRemoteFile]:
         """Yields files from the specified drive."""
         path_levels = [level for level in folder_path.split("/") if level]
         folder_path = "/".join(path_levels)
@@ -350,7 +368,7 @@ def drives(self) -> EntityCollection:
 
         return drives
 
-    def _get_shared_files_from_all_drives(self, parsed_drives):
+    def _get_shared_files_from_all_drives(self, parsed_drives) -> Iterable[MicrosoftSharePointRemoteFile]:
         drive_ids = [drive.id for drive in parsed_drives]
 
         shared_drive_items = execute_query_with_retry(self.one_drive_client.me.drive.shared_with_me())
@@ -361,7 +379,7 @@ def _get_shared_files_from_all_drives(self, parsed_drives):
             if parent_reference and parent_reference["driveId"] not in drive_ids:
                 yield from self._get_shared_drive_object(parent_reference["driveId"], drive_item.id, drive_item.web_url)
 
-    def get_all_files(self):
+    def get_all_files(self) -> Iterable[MicrosoftSharePointRemoteFile]:
         if self.config.search_scope in ("ACCESSIBLE_DRIVES", "ALL"):
             # Get files from accessible drives
             yield from self._get_files_by_drive_name(self.drives, self.config.folder_path)
@@ -383,17 +401,7 @@ def get_matching_files(self, globs: List[str], prefix: Optional[str], logger: lo
         files = self.get_all_files()
 
         files_generator = filter_http_urls(
-            self.filter_files_by_globs_and_start_date(
-                [
-                    MicrosoftSharePointRemoteFile(
-                        uri=path,
-                        download_url=download_url,
-                        last_modified=last_modified,
-                    )
-                    for path, download_url, last_modified in files
-                ],
-                globs,
-            ),
+            self.filter_files_by_globs_and_start_date([file for file in files], globs),
             logger,
         )
 
@@ -500,6 +508,7 @@ def upload(
                 filename=file_name,
                 bytes=file_size,
                 source_uri=file.uri,
+                created_at=file.created_at.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
                 updated_at=file.last_modified.strftime("%Y-%m-%dT%H:%M:%S.%fZ"),
             )
 

airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/stream_reader.py

@@ -29,6 +29,7 @@ class FolderNotFoundException(Exception):
 
 class MicrosoftSharePointRemoteFile(RemoteFile):
     download_url: str
+    created_at: datetime
 
 
 def filter_http_urls(files, logger):

airbyte-integrations/connectors/source-microsoft-sharepoint/source_microsoft_sharepoint/utils.py

@@ -30,7 +30,11 @@
 
 def create_mock_drive_item(is_file, name, children=None):
     """Helper function to create a mock drive item."""
-    mock_item = MagicMock(properties={"@microsoft.graph.downloadUrl": "test_url", "lastModifiedDateTime": "1991-08-24"})
+    mock_item = MagicMock(properties={
+        "@microsoft.graph.downloadUrl": "test_url", 
+        "lastModifiedDateTime": datetime(1991, 8, 24),
+        "createdDateTime": datetime(1991, 8, 24),
+    })
     mock_item.is_file = is_file
     mock_item.name = name
     mock_item.children.get.return_value.execute_query = Mock(return_value=children or [])
@@ -62,8 +66,18 @@ def create_mock_drive_files():
     Provides mock data for SharePoint drive files (personal drive).
     """
     return [
-        ("file1.csv", "https://example.com/file1.csv", datetime(2021, 1, 1)),
-        ("file2.txt", "https://example.com/file2.txt", datetime(2021, 1, 1)),
+        MicrosoftSharePointRemoteFile(
+            uri="file1.csv",
+            download_url="https://example.com/file1.csv",
+            last_modified=datetime(2021, 1, 1),
+            created_at=datetime(2021, 1, 1)
+        ),
+        MicrosoftSharePointRemoteFile(
+            uri="file2.txt",
+            download_url="https://example.com/file2.txt",
+            last_modified=datetime(2021, 1, 1),
+            created_at=datetime(2021, 1, 1)
+        ),
     ]
 
 
@@ -73,8 +87,18 @@ def create_mock_shared_drive_files():
     Provides mock data for SharePoint drive files (shared drives).
     """
     return [
-        ("file3.csv", "https://example.com/file3.csv", datetime(2021, 3, 1)),
-        ("file4.txt", "https://example.com/file4.txt", datetime(2021, 4, 1)),
+        MicrosoftSharePointRemoteFile(
+            uri="file3.csv",
+            download_url="https://example.com/file3.csv",
+            last_modified=datetime(2021, 3, 1),
+            created_at=datetime(2021, 3, 1)
+        ),
+        MicrosoftSharePointRemoteFile(
+            uri="file4.txt",
+            download_url="https://example.com/file4.txt",
+            last_modified=datetime(2021, 4, 1),
+            created_at=datetime(2021, 4, 1)
+        ),
     ]
 
 
@@ -258,6 +282,7 @@ def test_get_file(mock_requests_head, mock_requests_get, mock_get_access_token,
     file_uri = f"{file_uri}.{file_extension}"
     mock_file = Mock(download_url=f"https://example.com/file.{file_extension}", uri=file_uri)
     mock_file.last_modified = datetime(2021, 1, 1)
+    mock_file.created_at = datetime(2021, 1, 1)
     mock_logger = Mock()
     mock_get_access_token.return_value = "dummy_access_token"
 
@@ -381,8 +406,10 @@ def test_list_directories_and_files():
 
     assert len(result) == 2
     assert result == [
-        ("https://example.com/root/folder1/file1.txt", "test_url", "1991-08-24"),
-        ("https://example.com/root/file2.txt", "test_url", "1991-08-24"),
+        MicrosoftSharePointRemoteFile(uri='https://example.com/root/folder1/file1.txt', last_modified=datetime(1991, 8, 24, 0, 0),
+                                      mime_type=None, download_url='test_url', created_at=datetime(1991, 8, 24, 0, 0)),
+        MicrosoftSharePointRemoteFile(uri='https://example.com/root/file2.txt', last_modified=datetime(1991, 8, 24, 0, 0),
+                                      mime_type=None, download_url='test_url', created_at=datetime(1991, 8, 24, 0, 0)),
     ]
 
 
@@ -480,6 +507,7 @@ def test_get_shared_files_from_all_drives(
     "name": "TestFile.txt",
     "@microsoft.graph.downloadUrl": "http://example.com/download",
     "lastModifiedDateTime": "2021-01-01T00:00:00Z",
+    "createdDateTime": "2021-01-01T00:00:00Z"
 }
 
 empty_folder_response = {"folder": True, "value": []}
@@ -504,6 +532,7 @@ def test_get_shared_files_from_all_drives(
             "name": "NestedFile.txt",
             "@microsoft.graph.downloadUrl": "http://example.com/nested",
             "lastModifiedDateTime": "2021-01-02T00:00:00Z",
+            "createdDateTime": "2021-01-02T00:00:00Z"
         }
     ],
     "name": "subfolder2",
@@ -518,11 +547,7 @@ def test_get_shared_files_from_all_drives(
             file_response,
             [],
             [
-                (
-                    "http://example.com/TestFile.txt",
-                    "http://example.com/download",
-                    datetime.strptime("2021-01-01T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"),
-                )
+                MicrosoftSharePointRemoteFile(uri='http://example.com/TestFile.txt', last_modified=datetime(2021, 1, 1, 0, 0), mime_type=None, download_url='http://example.com/download', created_at=datetime(2021, 1, 1, 0, 0)),
             ],
             False,
             None,
@@ -539,11 +564,7 @@ def test_get_shared_files_from_all_drives(
                 not_empty_subfolder_response,
             ],
             [
-                (
-                    "http://example.com/subfolder2/NestedFile.txt",
-                    "http://example.com/nested",
-                    datetime.strptime("2021-01-02T00:00:00Z", "%Y-%m-%dT%H:%M:%SZ"),
-                )
+                MicrosoftSharePointRemoteFile(uri='http://example.com/subfolder2/NestedFile.txt', last_modified=datetime(2021, 1, 2, 0, 0), mime_type=None, download_url='http://example.com/nested', created_at=datetime(2021, 1, 2, 0, 0))
             ],
             False,
             None,