Introduce paging

lib/streaming_providers/base/manager.py

@@ -236,8 +236,21 @@ class ProviderManager:
     # VOD OPERATIONS (delegate to VodOperations)
     # ==========================================================================
 
-    def get_vod_node(self, provider_name: str, content_id: str = "", **kwargs) -> list:
-        return self.vod_ops.get_vod_node(provider_name, content_id=content_id, **kwargs)
+    def get_vod_node(
+        self,
+        provider_name: str,
+        content_id: str = "",
+        cursor: Optional[str] = None,
+        page_size: int = 24,
+        **kwargs,
+    ) -> dict:
+        return self.vod_ops.get_vod_node(
+            provider_name,
+            content_id=content_id,
+            cursor=cursor,
+            page_size=page_size,
+            **kwargs,
+        )
 
     def get_vod_manifest(self, provider_name: str, vod_id: str, **kwargs):
         return self.vod_ops.get_vod_manifest(provider_name, vod_id, **kwargs)

lib/streaming_providers/base/vod_operations.py

@@ -35,30 +35,67 @@ class VodOperations:
         self,
         provider_name: str,
         content_id: str = "",
+        cursor: Optional[str] = None,
+        page_size: int = 24,
         **kwargs,
-    ) -> List[Union[VodCategory, VodItem]]:
+    ) -> Dict:
         """
-        Return the children of a VOD node.
+        Return the children of a VOD node, with optional paging support.
 
         Args:
             provider_name: Provider to query.
             content_id:    Opaque node identifier from a previous response
                            (VodCategory.content_id).  Empty string → root.
+            cursor:        Opaque continuation token from a previous response's
+                           next_cursor field.  None → first page.
+                           Providers encode their own paging state (offset int,
+                           next-page URL, scroll token, etc.) in this string.
+            page_size:     Hint for how many entries to return per page.
+                           Providers may ignore or clamp this value.
 
         Returns:
-            Mixed list of VodCategory and VodItem entries.
+            {
+                "entries":     List[VodCategory | VodItem],
+                "next_cursor": Optional[str],   # None = no further pages
+                "total":       Optional[int],   # total count if known by provider
+            }
+            Providers that have not yet been updated to support paging may
+            return a plain list; this method normalises that into the dict
+            shape above with next_cursor=None so callers never need to
+            special-case the old return type.
 
         Raises:
             ValueError: Provider not found.
         """
         provider = self._get_provider(provider_name)
-        children = provider.get_vod_category(content_id=content_id, **kwargs)
         label = content_id or "root"
-        logger.info(
-            f"Retrieved {len(children)} VOD entries from '{provider_name}' "
-            f"at '{label}'"
+        logger.debug(
+            f"VodOperations: Fetching VOD node '{label}' from '{provider_name}' "
+            f"(cursor={cursor!r}, page_size={page_size})"
         )
-        return children
+
+        raw = provider.get_vod_category(
+            content_id=content_id,
+            cursor=cursor,
+            page_size=page_size,
+            **kwargs,
+        )
+
+        # Normalise: providers that have not yet adopted paging return a plain
+        # list.  Wrap it so all callers always get the same dict shape.
+        if isinstance(raw, list):
+            result = {"entries": raw, "next_cursor": None, "total": None}
+        else:
+            result = raw
+
+        entries = result.get("entries", [])
+        next_cursor = result.get("next_cursor")
+        logger.info(
+            f"VodOperations: Retrieved {len(entries)} VOD entries from '{provider_name}' "
+            f"at '{label}'"
+            + (f" — next_cursor present" if next_cursor else "")
+        )
+        return result
 
     def get_vod_manifest(
         self, provider_name: str, vod_id: str, **kwargs
@@ -93,6 +130,8 @@ class VodOperations:
         Get root VOD entries from all enabled providers that implement VOD.
 
         Providers that return implements_vod=False are silently skipped.
+        Only fetches the first page of root entries per provider — callers
+        that need subsequent pages should use get_vod_node directly.
         """
         enabled = self.registry.get_enabled_providers()
         logger.info(f"Fetching VOD roots from {len(enabled)} providers")
@@ -105,7 +144,8 @@ class VodOperations:
                 provider = self.registry.get_provider(name)
                 if not getattr(provider, "implements_vod", False):
                     continue
-                entries = self.get_vod_node(name, [])
+                node = self.get_vod_node(name, content_id="")
+                entries = node["entries"]
                 result[name] = entries
                 total += len(entries)
             except Exception as e:

lib/streaming_providers/providers/magenta2/provider.py

@@ -1119,6 +1119,13 @@ class Magenta2Provider(StreamingProvider):
             content_id: Opaque node identifier produced by a previous
                         get_vod_category call (e.g. "lane:322341",
                         "series:GN_SERIES_20914057").  Empty string → root.
+            cursor:     Opaque continuation token from a previous response's
+                        next_cursor field.  None → first page.
+                        Encoded as a plain integer string by VodManager
+                        (the $offset value for the next UnstructuredGrid call).
+            page_size:  Number of items to request per page (default from
+                        VOD_DEFAULT_PAGE_SIZE).  Passed straight through to
+                        VodManager and then to the $size query parameter.
         """
         if not self._vod_manager:
             raise RuntimeError("VodManager not available - configuration discovery may have failed")

lib/streaming_providers/providers/magenta2/vod_manager.py

@@ -38,8 +38,8 @@ tvhubs base URL resolution order
 
 Public interface
 -----------------
-    vod_manager.get_children(content_id, **kwargs)
-        -> List[VodCategory | VodItem]
+    vod_manager.get_children(content_id, *, cursor, page_size, **kwargs)
+        -> Dict with keys: entries, next_cursor, total
 """
 
 import time
@@ -168,9 +168,10 @@ class VodManager:
         self,
         content_id: str,
         *,
+        cursor: Optional[str] = None,
         page_size: int = VOD_DEFAULT_PAGE_SIZE,
         offset: int = 0,
-    ) -> List[Union[VodCategory, VodItem]]:
+    ) -> Dict:
         """
         Return the children of a VOD node identified by *content_id*.
 
@@ -187,14 +188,48 @@ class VodManager:
 
         Args:
             content_id: Opaque node identifier.
+            cursor:     Opaque continuation token returned in a previous
+                        response's next_cursor field.  For lane/UnstructuredGrid
+                        nodes, VodManager encodes the next $offset as a plain
+                        integer string (e.g. "24", "48").  None → first page.
+                        For non-paginated nodes (home, series, seasons) the
+                        cursor is ignored and next_cursor is always None.
             page_size:  Items per page for lane fetches.
-            offset:     Pagination offset for lane fetches.
+            offset:     Direct offset override for internal callers that bypass
+                        the cursor mechanism (e.g. _resolve_gn_id_to_media_id).
+                        Ignored when cursor is supplied.
+
+        Returns:
+            {
+                "entries":     List[VodCategory | VodItem],
+                "next_cursor": Optional[str],   # None when no further pages exist
+                "total":       Optional[int],   # total item count if known by API
+            }
         """
-        logger.debug(f"{self._provider}: get_children content_id={content_id!r}")
+        logger.debug(
+            f"{self._provider}: get_children content_id={content_id!r} "
+            f"cursor={cursor!r} page_size={page_size}"
+        )
+
+        # Decode cursor → offset.  cursor takes precedence over the legacy
+        # offset kwarg so that route-layer callers always use cursor.
+        if cursor is not None:
+            try:
+                offset = int(cursor)
+            except (ValueError, TypeError):
+                logger.warning(
+                    f"{self._provider}: Invalid cursor value {cursor!r}, "
+                    "ignoring and starting from offset 0"
+                )
+                offset = 0
+
         params = self._base_params()
 
         if not content_id:
-            return self._fetch_home_lanes(params)
+            # Home lanes are not paginated — always return first page wrapped
+            # in the standard dict so callers never need to branch on type.
+            lanes = self._fetch_home_lanes(params)
+            return {"entries": lanes, "next_cursor": None, "total": None}
 
         # ── Lane (UnstructuredGrid) ──────────────────────────────────────
         if content_id.startswith("lane:"):
@@ -217,21 +252,25 @@ class VodManager:
         # ── Series ──────────────────────────────────────────────────────
         if content_id.startswith("series:"):
             gn_id = content_id[len("series:"):]
-            return self._fetch_series_seasons(gn_id, params)
+            seasons = self._fetch_series_seasons(gn_id, params)
+            return {"entries": seasons, "next_cursor": None, "total": None}
 
         # ── Season ──────────────────────────────────────────────────────
         if content_id.startswith("season:"):
             gn_id = content_id[len("season:"):]
-            return self._fetch_season_episodes(gn_id, params)
+            episodes = self._fetch_season_episodes(gn_id, params)
+            return {"entries": episodes, "next_cursor": None, "total": None}
 
         # ── Episode / Movie ─────────────────────────────────────────────
         if content_id.startswith("episode:"):
             gn_id = content_id[len("episode:"):]
-            return self._fetch_single_episode(gn_id, params)
+            items = self._fetch_single_episode(gn_id, params)
+            return {"entries": items, "next_cursor": None, "total": None}
 
         if content_id.startswith("movie:"):
             gn_id = content_id[len("movie:"):]
-            return self._fetch_single_item(gn_id, params)
+            items = self._fetch_single_item(gn_id, params)
+            return {"entries": items, "next_cursor": None, "total": None}
 
         # ── Legacy / backwards-compat ───────────────────────────────────
         # Support old-style content_ids (GN_SERIES_*, GN_SEASON_*, etc.)
@@ -248,12 +287,16 @@ class VodManager:
         if node_id.startswith("VodDetails/"):
             node_id = node_id.split("/")[-1]
         if node_id.startswith(VOD_PREFIX_SEASON):
-            return self._fetch_season_episodes(node_id, params)
+            episodes = self._fetch_season_episodes(node_id, params)
+            return {"entries": episodes, "next_cursor": None, "total": None}
         if node_id.startswith(VOD_PREFIX_SERIES):
-            return self._fetch_series_seasons(node_id, params)
+            seasons = self._fetch_series_seasons(node_id, params)
+            return {"entries": seasons, "next_cursor": None, "total": None}
         if node_id.startswith(VOD_PREFIX_EPISODE):
-            return self._fetch_single_episode(node_id, params)
-        return self._fetch_single_item(node_id, params)
+            items = self._fetch_single_episode(node_id, params)
+            return {"entries": items, "next_cursor": None, "total": None}
+        items = self._fetch_single_item(node_id, params)
+        return {"entries": items, "next_cursor": None, "total": None}
 
     # =========================================================================
     # Private helpers – HTTP layer
@@ -731,7 +774,7 @@ class VodManager:
         offset: int = 0,
         fetch_url: Optional[str] = None,
         extra_params: Optional[Dict] = None,
-    ) -> List[Union[VodCategory, VodItem]]:
+    ) -> Dict:
         """
         Fetch items from an UnstructuredGrid lane.
 
@@ -743,6 +786,13 @@ class VodManager:
                           together in one clean dict.
             extra_params: Additional params from the registry (merged after
                           fetch_url params so they take precedence).
+
+        Returns:
+            {
+                "entries":     List[VodCategory | VodItem],
+                "next_cursor": Optional[str],   # str(next_offset) or None
+                "total":       Optional[int],   # total item count if returned by API
+            }
         """
         from urllib.parse import urlparse, parse_qs, urlunparse
 
@@ -770,7 +820,7 @@ class VodManager:
         )
         data = self._get(url, paged_params)
         if not data:
-            return []
+            return {"entries": [], "next_cursor": None, "total": None}
 
         content = data.get("content", {})
         results: List[Union[VodCategory, VodItem]] = []
@@ -779,12 +829,27 @@ class VodManager:
             if node is not None:
                 results.append(node)
 
-        total = content.get("page", {}).get("total", len(results))
+        page_info = content.get("page", {})
+        total: Optional[int] = page_info.get("total")
+
+        # Compute next_cursor.  Use the API-reported total when available so
+        # we never request a page beyond the end.  Fall back to the heuristic
+        # that a full page means there are probably more items.
+        next_offset = offset + len(results)
+        if total is not None:
+            next_cursor: Optional[str] = str(next_offset) if next_offset < total else None
+        else:
+            # No total from API: assume there are more pages if we received a
+            # full page; stop if we received fewer items than requested.
+            next_cursor = str(next_offset) if len(results) >= page_size else None
+
         logger.debug(
-            f"{self._provider}: Lane {content_id} – fetched {len(results)}/{total} items "
-            f"(offset={offset})"
+            f"{self._provider}: Lane {content_id} – fetched {len(results)}"
+            + (f"/{total}" if total is not None else "")
+            + f" items (offset={offset})"
+            + (f" → next_cursor={next_cursor!r}" if next_cursor else " → end of lane")
         )
-        return results
+        return {"entries": results, "next_cursor": next_cursor, "total": total}
 
     def _map_unstructured_item(
         self, item: Dict, params: Dict

routes/vod.py

@@ -12,15 +12,23 @@ GET /api/providers/<provider>/vod/<path:path>
     Navigates the VOD tree by URL-safe path segments.
     e.g. /api/providers/discovery_de/vod/sports/nordic-combined
 
+    Query parameters:
+        cursor    Opaque continuation token returned in a previous response's
+                  next_cursor field.  Omit (or pass empty) for the first page.
+        size      Number of entries to request per page (default: 24).
+                  Providers may clamp or ignore this value.
+
     Response:
     {
-        "provider": "discovery_de",
-        "path": "sports/nordic-combined",
+        "provider":    "discovery_de",
+        "content_id":  "sports/nordic-combined",
         "entries": [
             {"type": "vod_category", "id": "...", "name": "...", "slug": "..."},
             {"type": "vod",          "id": "...", "name": "...", "slug": "..."}
         ],
-        "count": 12
+        "count":       12,
+        "next_cursor": "<opaque>",   # null when no further pages exist
+        "total":       120           # null when provider does not expose total
     }
 
 Stream / manifest / DRM endpoints for VodItems are in streams.py:
@@ -31,7 +39,7 @@ These are registered in streams.py (same pattern as channels and events)
 and must be set up BEFORE setup_vod_routes() so Bottle matches them first.
 """
 
-from bottle import response
+from bottle import request, response
 from streaming_providers.base.utils import logger
 
 
@@ -40,10 +48,32 @@ def setup_vod_routes(app, manager):
     def _serialize(entries) -> list:
         return [e.to_dict() for e in entries]
 
+    def _parse_paging_params() -> tuple[str | None, int]:
+        """
+        Extract and validate paging query parameters from the current request.
+
+        Returns:
+            (cursor, page_size) — cursor is None when not supplied or empty.
+        """
+        cursor = request.query.get("cursor") or None
+        try:
+            page_size = int(request.query.get("size", 24))
+            if page_size < 1:
+                page_size = 24
+        except (ValueError, TypeError):
+            page_size = 24
+        return cursor, page_size
+
     @app.route("/api/providers/<provider>/vod", method="GET")
     def get_vod_root(provider):
+        cursor, page_size = _parse_paging_params()
         try:
-            entries = manager.get_vod_node(provider_name=provider, content_id="")
+            result = manager.get_vod_node(
+                provider_name=provider,
+                content_id="",
+                cursor=cursor,
+                page_size=page_size,
+            )
         except ValueError as e:
             response.status = 404
             return {"error": "Provider not found", "message": str(e), "provider": provider}
@@ -51,9 +81,16 @@ def setup_vod_routes(app, manager):
             logger.error(f"Failed to get VOD root from provider: {e}")
             response.status = 500
             return {"error": "Failed to get VOD entries", "message": str(e), "provider": provider}
-        serialized = _serialize(entries)
+        serialized = _serialize(result["entries"])
         response.status = 200
-        return {"provider": provider, "content_id": "", "entries": serialized, "count": len(serialized)}
+        return {
+            "provider": provider,
+            "content_id": "",
+            "entries": serialized,
+            "count": len(serialized),
+            "next_cursor": result.get("next_cursor"),
+            "total": result.get("total"),
+        }
 
     @app.route("/api/providers/<provider>/vod/<content_id:path>", method="GET")
     def get_vod_node(provider, content_id):
@@ -73,10 +110,13 @@ def setup_vod_routes(app, manager):
         if not content_id:
             return get_vod_root(provider)
 
+        cursor, page_size = _parse_paging_params()
         try:
-            entries = manager.get_vod_node(
+            result = manager.get_vod_node(
                 provider_name=provider,
                 content_id=content_id,
+                cursor=cursor,
+                page_size=page_size,
             )
         except ValueError as e:
             response.status = 404
@@ -85,6 +125,13 @@ def setup_vod_routes(app, manager):
             logger.error(f"Failed to get VOD node from provider: {e}")
             response.status = 500
             return {"error": "Failed to get VOD entries", "message": str(e), "provider": provider, "content_id": content_id}
-        serialized = _serialize(entries)
+        serialized = _serialize(result["entries"])
         response.status = 200
-        return {"provider": provider, "content_id": content_id, "entries": serialized, "count": len(serialized)}
+        return {
+            "provider": provider,
+            "content_id": content_id,
+            "entries": serialized,
+            "count": len(serialized),
+            "next_cursor": result.get("next_cursor"),
+            "total": result.get("total"),
+        }