Merge pull request #1170 from Hestia-Homes/bug/pashub-fetcher-correctly-writes-to-sharepoint

Pashub fetcher bugs: Playwright /tmp cleanup, and sharepoint folder pagination

backend/pashub_fetcher/tests/test_token_getter.py

@@ -0,0 +1,50 @@
+from unittest.mock import MagicMock, patch
+
+from backend.pashub_fetcher.token_getter import get_token_from_local_storage
+
+
+def _configure_playwright_mock(mock_sync_playwright: MagicMock) -> None:
+    mock_page = MagicMock()
+    mock_page.url = "https://pashub.net/dashboard"
+    mock_page.evaluate.return_value = "fake-token"
+
+    mock_context = MagicMock()
+    mock_context.new_page.return_value = mock_page
+
+    mock_browser = MagicMock()
+    mock_browser.new_context.return_value = mock_context
+
+    mock_p = MagicMock()
+    mock_p.chromium.launch.return_value = mock_browser
+
+    mock_sync_playwright.return_value.__enter__.return_value = mock_p
+
+
+@patch("backend.pashub_fetcher.token_getter.shutil.rmtree")
+@patch("backend.pashub_fetcher.token_getter.glob.glob")
+@patch("backend.pashub_fetcher.token_getter.sync_playwright")
+def test_playwright_tmp_dirs_are_cleaned_up_after_browser_close(
+    mock_sync_playwright: MagicMock,
+    mock_glob: MagicMock,
+    mock_rmtree: MagicMock,
+) -> None:
+    # Arrange
+    fake_artifacts = ["/tmp/playwright-artifacts-abc12"]
+    fake_profiles = ["/tmp/playwright_chromiumdev_profile-xyz99"]
+
+    def glob_side_effect(pattern: str) -> list[str]:
+        if "playwright-artifacts-*" in pattern:
+            return fake_artifacts
+        if "playwright_chromiumdev_profile-*" in pattern:
+            return fake_profiles
+        return []
+
+    mock_glob.side_effect = glob_side_effect
+    _configure_playwright_mock(mock_sync_playwright)
+
+    # Act
+    get_token_from_local_storage("user@example.com", "secret")
+
+    # Assert
+    mock_rmtree.assert_any_call("/tmp/playwright-artifacts-abc12", ignore_errors=True)
+    mock_rmtree.assert_any_call("/tmp/playwright_chromiumdev_profile-xyz99", ignore_errors=True)

backend/pashub_fetcher/token_getter.py

@@ -1,4 +1,6 @@
+import glob
 import os
+import shutil
 
 from playwright.sync_api import sync_playwright, TimeoutError as PlaywrightTimeoutError
 
@@ -90,5 +92,12 @@ def get_token_from_local_storage(
             context.close()
             browser.close()
 
+            for pattern in (
+                "/tmp/playwright-artifacts-*",
+                "/tmp/playwright_chromiumdev_profile-*",
+            ):
+                for path in glob.glob(pattern):
+                    shutil.rmtree(path, ignore_errors=True)
+
             if record_video and video_dir:
                 logger.info(f"Video(s) saved in: {video_dir}")

utils/sharepoint/domna_sharepoint_client.py

@@ -37,7 +37,7 @@ class DomnaSharepointClient:
             site_id=self.sharepoint_drive.value,
         )
 
-        return sharepoint_client.list_folder_contents(path)
+        return sharepoint_client.list_folder_contents(path, page_size=500)
 
     def does_folder_exists_at(self, file_name: str, file_path: str):
         folders: Dict[str, Any] = self.get_folders_in_path(file_path)