fix: unify streaming download chunk size with dedicated DOWNLOAD_CHUNK_SIZE_MB knob

CHANGELOG.md

@@ -5,6 +5,21 @@
   hierarchy path (e.g. `"Marketing/Q1 Reports"`). The walk is performed level by
   level using the REST API name filter, so a path with *n* components issues *n*
   requests. Returns the matching `ProjectItem` or `None` if no project is found.
+* Unified streaming download chunk size for the file-backed download paths --
+  `views.populate_csv` and `views.populate_excel`, the CSV/Excel branches of
+  `custom_views.*`, and `workbooks.download` / `datasources.download` /
+  `flows.download`. Previously the mixed 1024-byte and 10240-byte chunks caused
+  multi-second latency for large view exports. These paths now use a dedicated
+  `DOWNLOAD_CHUNK_SIZE_MB` config value (default 1 MB, overridable via the
+  `TSC_DOWNLOAD_CHUNK_SIZE_MB` env var). Upload / chunked-publish continues to
+  use `CHUNK_SIZE_MB` (default 50 MB, overridable via `TSC_CHUNK_SIZE_MB`) --
+  they are separate knobs because a large read chunk delays first-byte yield on
+  slow connections while a large write chunk reduces per-request overhead.
+  Behavior note: callers that previously streamed 1 KB at a time will now hold
+  up to 1 MB resident per chunk; memory-constrained callers can drop this via
+  the env var. Follow-up: `views.populate_pdf` and `views.populate_image` still
+  buffer the full response in memory via `server_response.content` and are not
+  covered by this change.
 * Preserve HTTP method and body across 3xx redirects. Previously `requests`
   followed 301/302/303 by converting POST to GET and dropping the body, so
   endpoints like `users.add`, `workbooks.publish`, and any write hitting a

tableauserverclient/config.py

@@ -14,10 +14,40 @@ def FILESIZE_LIMIT_MB(self):
         return min(int(os.getenv("TSC_FILESIZE_LIMIT_MB", 64)), 64)
 
     # For when a datasource is over 64MB, break it into 5MB(standard chunk size) chunks
+    # Applies to *upload* / chunked publish. Downloads use DOWNLOAD_CHUNK_SIZE_MB.
     @property
     def CHUNK_SIZE_MB(self):
         return int(os.getenv("TSC_CHUNK_SIZE_MB", 5 * 10))  # 5MB felt too slow, upped it to 50
 
+    # Chunk size for streaming *downloads* (view CSV / Excel, workbook /
+    # datasource / flow downloads). Kept separate from the upload knob because
+    # a large read chunk delays the first-byte yield on slow connections --
+    # requests.iter_content buffers up to chunk_size before yielding, so on a
+    # 1 Mbps link a 50 MB chunk means ~7 minutes before the first yield.
+    # 1 MB is empirically a reasonable balance between per-chunk overhead and
+    # progressive-yield latency; callers who want a different tradeoff can
+    # tune via TSC_DOWNLOAD_CHUNK_SIZE_MB.
+    #
+    # No upper bound is enforced, but very large values (thousands of MB) will
+    # OOM on constrained hosts because each chunk is buffered in memory before
+    # it is written or yielded. Keep this under ~100 MB unless the caller has
+    # specifically measured a benefit at higher values.
+    #
+    # Bounds: values <= 0 or non-numeric input are treated as invalid and fall
+    # back to the 1 MB default; 0 or a negative chunk size would silently
+    # corrupt the output because iter_content interprets it as "read all".
+    _DOWNLOAD_CHUNK_SIZE_MB_DEFAULT = 1
+
+    @property
+    def DOWNLOAD_CHUNK_SIZE_MB(self) -> int:
+        raw = os.getenv("TSC_DOWNLOAD_CHUNK_SIZE_MB", str(self._DOWNLOAD_CHUNK_SIZE_MB_DEFAULT))
+        try:
+            value = int(raw)
+        except ValueError:
+            # invalid env value; fall back to default
+            value = self._DOWNLOAD_CHUNK_SIZE_MB_DEFAULT
+        return max(1, value)
+
     # Default page size
     @property
     def PAGE_SIZE(self):

tableauserverclient/server/endpoint/custom_views_endpoint.py

@@ -1,14 +1,13 @@
 import io
 import logging
 import os
-from contextlib import closing
 from pathlib import Path
 from typing import TYPE_CHECKING
 from collections.abc import Iterator
 
 from tableauserverclient.config import BYTES_PER_MB, config
 from tableauserverclient.filesys_helpers import get_file_object_size
-from tableauserverclient.server.endpoint.endpoint import QuerysetEndpoint, api
+from tableauserverclient.server.endpoint.endpoint import DownloadableMixin, QuerysetEndpoint, api
 from tableauserverclient.server.endpoint.exceptions import MissingRequiredFieldError
 from tableauserverclient.models import CustomViewItem, PaginationItem
 from tableauserverclient.server import (
@@ -42,7 +41,7 @@
 io_types_w = (io.BufferedWriter, io.BytesIO)
 
 
-class CustomViews(QuerysetEndpoint[CustomViewItem]):
+class CustomViews(QuerysetEndpoint[CustomViewItem], DownloadableMixin):
     def __init__(self, parent_srv):
         super().__init__(parent_srv)
 
@@ -229,9 +228,7 @@ def _get_custom_view_csv(
         self, custom_view_item: CustomViewItem, req_options: "CSVRequestOptions | None"
     ) -> Iterator[bytes]:
         url = f"{self.baseurl}/{custom_view_item.id}/data"
-
-        with closing(self.get_request(url, request_object=req_options, parameters={"stream": True})) as server_response:
-            yield from server_response.iter_content(1024)
+        return self._stream_content(url, req_options)
 
     @api(version="3.18")
     def update(self, view_item: CustomViewItem) -> CustomViewItem | None:

tableauserverclient/server/endpoint/endpoint.py

@@ -10,6 +10,7 @@
 from packaging.version import Version
 from functools import wraps
 from xml.etree.ElementTree import ParseError
+from collections.abc import Iterator
 from typing import (
     Any,
     Callable,
@@ -21,8 +22,9 @@
 )
 from typing_extensions import Self
 
+from tableauserverclient.config import BYTES_PER_MB, config
 from tableauserverclient.models.pagination_item import PaginationItem
-from tableauserverclient.server.request_options import RequestOptions
+from tableauserverclient.server.request_options import RequestOptions, RequestOptionsBase
 from tableauserverclient.filesys_helpers import to_filename, make_download_path
 
 from tableauserverclient.server.endpoint.exceptions import (
@@ -461,9 +463,13 @@ def wrapper(self: E, *args: P.args, **kwargs: P.kwargs) -> R:
 class DownloadableMixin:
     """Mixin for endpoints whose resources can be downloaded as binary files.
 
-    Provides a single private helper that streams a server response to a file
-    path or writable file object, avoiding copy-paste of the identical streaming
-    loop in Workbooks, Datasources, and Flows.
+    Provides two private helpers, avoiding copy-paste of the streaming loop
+    across endpoints:
+
+    - _download_content streams a response to a file path or writable object
+      (used by Workbooks, Datasources, Flows).
+    - _stream_content yields chunks for callers that want to materialize the
+      body in-memory or process it lazily (used by Views, CustomViews).
     """
 
     def _download_content(
@@ -491,18 +497,36 @@ def _download_content(
             m = Message()
             m["Content-Disposition"] = server_response.headers["Content-Disposition"]
             filename = m.get_filename(failobj="")
+            chunk_size = config.DOWNLOAD_CHUNK_SIZE_MB * BYTES_PER_MB
             if isinstance(filepath, _io_types_w):
-                for chunk in server_response.iter_content(1024):  # 1KB
+                for chunk in server_response.iter_content(chunk_size):
                     filepath.write(chunk)
                 return filepath
             else:
                 filename = to_filename(os.path.basename(filename))
                 download_path = make_download_path(filepath, filename)
                 with open(download_path, "wb") as f:
-                    for chunk in server_response.iter_content(1024):  # 1KB
+                    for chunk in server_response.iter_content(chunk_size):
                         f.write(chunk)
                 return os.path.abspath(download_path)
 
+    def _stream_content(
+        self,
+        url: str,
+        request_object: RequestOptionsBase | None = None,
+    ) -> Iterator[bytes]:
+        """Stream content at url as chunks.
+
+        Suitable for callers that want to materialize the body in-memory
+        (e.g. b"".join(iterator)) or process it lazily as a stream, rather than
+        write it to a file. Complements _download_content, which writes to disk.
+        """
+        chunk_size = config.DOWNLOAD_CHUNK_SIZE_MB * BYTES_PER_MB
+        with closing(
+            self.get_request(url, request_object=request_object, parameters={"stream": True})  # type: ignore[attr-defined]
+        ) as server_response:
+            yield from server_response.iter_content(chunk_size)
+
 
 class QuerysetEndpoint(Endpoint, Generic[T]):
     @api(version="2.0")

tableauserverclient/server/endpoint/views_endpoint.py

@@ -1,8 +1,7 @@
 import logging
-from contextlib import closing
 
 from tableauserverclient.models.permissions_item import PermissionsRule
-from tableauserverclient.server.endpoint.endpoint import QuerysetEndpoint, api
+from tableauserverclient.server.endpoint.endpoint import DownloadableMixin, QuerysetEndpoint, api
 from tableauserverclient.server.endpoint.exceptions import MissingRequiredFieldError, UnsupportedAttributeError
 from tableauserverclient.server.endpoint.permissions_endpoint import _PermissionsEndpoint
 from tableauserverclient.server.endpoint.resource_tagger import TaggingMixin
@@ -25,7 +24,7 @@
     )
 
 
-class Views(QuerysetEndpoint[ViewItem], TaggingMixin[ViewItem]):
+class Views(QuerysetEndpoint[ViewItem], TaggingMixin[ViewItem], DownloadableMixin):
     """
     The Tableau Server Client provides methods for interacting with view
     resources, or endpoints. These methods correspond to the endpoints for views
@@ -262,9 +261,7 @@ def csv_fetcher():
 
     def _get_view_csv(self, view_item: ViewItem, req_options: "CSVRequestOptions | None") -> Iterator[bytes]:
         url = f"{self.baseurl}/{view_item.id}/data"
-
-        with closing(self.get_request(url, request_object=req_options, parameters={"stream": True})) as server_response:
-            yield from server_response.iter_content(1024)
+        return self._stream_content(url, req_options)
 
     @api(version="3.8")
     def populate_excel(self, view_item: ViewItem, req_options: "ExcelRequestOptions | None" = None) -> None:
@@ -301,9 +298,7 @@ def excel_fetcher():
 
     def _get_view_excel(self, view_item: ViewItem, req_options: "ExcelRequestOptions | None") -> Iterator[bytes]:
         url = f"{self.baseurl}/{view_item.id}/crosstab/excel"
-
-        with closing(self.get_request(url, request_object=req_options, parameters={"stream": True})) as server_response:
-            yield from server_response.iter_content(1024)
+        return self._stream_content(url, req_options)
 
     @api(version="3.2")
     def populate_permissions(self, item: ViewItem) -> None:

test/test_view.py

@@ -328,6 +328,59 @@ def test_populate_csv_default_maxage(server: TSC.Server) -> None:
         assert response == csv_file
 
 
+def test_stream_content_uses_configured_chunk_size(server: TSC.Server, monkeypatch) -> None:
+    # Regression guard: the shared DownloadableMixin._stream_content path must
+    # stream with config.DOWNLOAD_CHUNK_SIZE_MB * BYTES_PER_MB, not the pre-fix
+    # 1024 bytes.
+    from tableauserverclient.config import BYTES_PER_MB, config
+
+    captured: list[int] = []
+    real_iter_content = None
+
+    def spy_iter_content(self, chunk_size=None, decode_unicode=False):
+        captured.append(chunk_size)
+        assert real_iter_content is not None
+        # Pass chunk_size / decode_unicode by name so a signature reorder in
+        # requests fails loudly here instead of silently binding to the wrong
+        # parameter.
+        return real_iter_content(self, chunk_size=chunk_size, decode_unicode=decode_unicode)
+
+    import requests
+
+    real_iter_content = requests.Response.iter_content
+    monkeypatch.setattr(requests.Response, "iter_content", spy_iter_content)
+
+    response = POPULATE_CSV.read_bytes()
+    with requests_mock.mock() as m:
+        m.get(server.views.baseurl + "/d79634e1-6063-4ec9-95ff-50acbf609ff5/data", content=response)
+        single_view = TSC.ViewItem()
+        single_view._id = "d79634e1-6063-4ec9-95ff-50acbf609ff5"
+        server.views.populate_csv(single_view)
+        # Materialize the iterator so iter_content is actually invoked.
+        b"".join(single_view.csv)
+
+    assert captured, "iter_content was never invoked"
+    assert captured[0] == config.DOWNLOAD_CHUNK_SIZE_MB * BYTES_PER_MB
+
+
+def test_download_chunk_size_clamps_zero(monkeypatch) -> None:
+    # TSC_DOWNLOAD_CHUNK_SIZE_MB=0 would cause requests.iter_content to read the
+    # entire response as a single chunk (defeating the streaming behavior); make
+    # sure the config clamps to the 1 MB default instead.
+    from tableauserverclient.config import config
+
+    monkeypatch.setenv("TSC_DOWNLOAD_CHUNK_SIZE_MB", "0")
+    assert config.DOWNLOAD_CHUNK_SIZE_MB == 1
+
+
+def test_download_chunk_size_rejects_non_numeric(monkeypatch) -> None:
+    # Non-numeric env values must not crash the download path; fall back to 1 MB.
+    from tableauserverclient.config import config
+
+    monkeypatch.setenv("TSC_DOWNLOAD_CHUNK_SIZE_MB", "abc")
+    assert config.DOWNLOAD_CHUNK_SIZE_MB == 1
+
+
 def test_populate_image_missing_id(server: TSC.Server) -> None:
     single_view = TSC.ViewItem()
     single_view._id = None

test/test_workbook.py

@@ -339,6 +339,39 @@ def test_download_object(server: TSC.Server) -> None:
             assert isinstance(file_path, BytesIO)
 
 
+def test_download_uses_configured_chunk_size(server: TSC.Server, tmp_path: Path, monkeypatch) -> None:
+    # Regression guard: the shared DownloadableMixin._download_content path must
+    # stream with config.DOWNLOAD_CHUNK_SIZE_MB * BYTES_PER_MB, not the pre-fix
+    # 1024 bytes.
+    from tableauserverclient.config import BYTES_PER_MB, config
+
+    captured: list[int] = []
+    real_iter_content = None
+
+    def spy_iter_content(self, chunk_size=None, decode_unicode=False):
+        captured.append(chunk_size)
+        assert real_iter_content is not None
+        # Pass chunk_size / decode_unicode by name so a signature reorder in
+        # requests fails loudly here instead of silently binding to the wrong
+        # parameter.
+        return real_iter_content(self, chunk_size=chunk_size, decode_unicode=decode_unicode)
+
+    import requests
+
+    real_iter_content = requests.Response.iter_content
+    monkeypatch.setattr(requests.Response, "iter_content", spy_iter_content)
+
+    with requests_mock.mock() as m:
+        m.get(
+            server.workbooks.baseurl + "/1f951daf-4061-451a-9df1-69a8062664f2/content",
+            headers={"Content-Disposition": 'name="tableau_workbook"; filename="RESTAPISample.twbx"'},
+        )
+        server.workbooks.download("1f951daf-4061-451a-9df1-69a8062664f2", filepath=tmp_path)
+
+    assert captured, "iter_content was never invoked"
+    assert captured[0] == config.DOWNLOAD_CHUNK_SIZE_MB * BYTES_PER_MB
+
+
 def test_download_sanitizes_name(server: TSC.Server, tmp_path: Path) -> None:
     filename = "Name,With,Commas.twbx"
     disposition = f'name="tableau_workbook"; filename="{filename}"'