diff --git a/tableauserverclient/helpers/strings.py b/tableauserverclient/helpers/strings.py index b1c868cc7..63a3193f7 100644 --- a/tableauserverclient/helpers/strings.py +++ b/tableauserverclient/helpers/strings.py @@ -7,23 +7,47 @@ # create the replacement text with the correct type T = TypeVar("T", str, bytes) +_REDACTED = "********" -def _redact_any_type(xml: T, sensitive_word: T, replacement: T, encoding=None) -> T: +# Elements that carry sign-in or connection credentials. Only the attributes named here are kept on them; +# any other attribute, including ones the server or library adds later, is replaced. +_CREDENTIAL_ELEMENTS = {"credentials", "connectionCredentials"} +_SAFE_CREDENTIAL_ATTRIBUTES = {"name", "personalAccessTokenName", "isUat", "embed", "oAuth", "id"} + +# Outside credential elements, attributes and elements whose names contain one of these are replaced. +_SENSITIVE_NAME_PARTS = ("password", "secret", "token", "jwt", "authorization") + + +def _local_name(name: str) -> str: + return name.rsplit("}", 1)[-1] + + +def _is_sensitive(name: str) -> bool: + lowered = name.lower() + return any(part in lowered for part in _SENSITIVE_NAME_PARTS) + + +def _redact_any_type(xml: T, replacement: T, encoding=None) -> T: try: root = fromstring(xml) - matches = root.findall(".//*[@password]") - for item in matches: - item.attrib["password"] = "********" - matches = root.findall(".//password") - for item in matches: - item.text = "********" - # tostring returns bytes unless an encoding value is passed; cast since - # the callers pass encoding="unicode" for str and None for bytes - return cast(T, tostring(root, encoding=encoding)) except Exception: - # something about the xml handling failed. Just cut off the text at the first occurrence of "password" - location = xml.find(sensitive_word) - return xml[:location] + replacement + # Not parseable (truncated, multipart, not xml): we can't tell which parts are safe, so log none of it. + return replacement + for element in root.iter(): + tag = _local_name(element.tag) + is_credential_element = tag in _CREDENTIAL_ELEMENTS + for attribute in element.attrib: + name = _local_name(attribute) + if is_credential_element: + if name not in _SAFE_CREDENTIAL_ATTRIBUTES: + element.attrib[attribute] = _REDACTED + elif _is_sensitive(name): + element.attrib[attribute] = _REDACTED + if element.text and _is_sensitive(tag): + element.text = _REDACTED + # tostring returns bytes unless an encoding value is passed; cast since + # the callers pass encoding="unicode" for str and None for bytes + return cast(T, tostring(root, encoding=encoding)) @singledispatch @@ -34,13 +58,13 @@ def redact_xml(content): @redact_xml.register def _(xml: str) -> str: - out = _redact_any_type(xml, "password", "...[redacted]", encoding="unicode") + out = _redact_any_type(xml, "[unparseable content redacted]", encoding="unicode") return out @redact_xml.register # type: ignore[no-redef] def _(xml: bytes) -> bytes: - return cast(bytes, _redact_any_type(xml, b"password", b"..[redacted]")) + return cast(bytes, _redact_any_type(xml, b"[unparseable content redacted]")) @overload diff --git a/tableauserverclient/server/endpoint/endpoint.py b/tableauserverclient/server/endpoint/endpoint.py index 10d3cba84..838a21d5c 100644 --- a/tableauserverclient/server/endpoint/endpoint.py +++ b/tableauserverclient/server/endpoint/endpoint.py @@ -150,10 +150,9 @@ def _make_request( parameters.setdefault("allow_redirects", False) logger.debug(f"request method {method.__name__}, url: {url}") - if content: - redacted = helpers.strings.redact_xml(content[:200]) - # this needs to be under a trace or something, it's a LOT - # logger.debug("request content: {}".format(redacted)) + # this needs to be under a trace or something, it's a LOT + # if content: + # logger.debug("request content: {}".format(helpers.strings.redact_xml(content[:200]))) # a request can, for stuff like publishing, spin for ages waiting for a response. # we need some user-facing activity so they know it's not dead. @@ -175,11 +174,10 @@ def _make_request( server_response, url = self._follow_redirect_if_any(method, url, parameters, server_response) self._check_status(server_response, url) - loggable_response = self.log_response_safely(server_response) logger.debug(f"Server response from {url}") # uncomment the following to log full responses in debug mode # BE CAREFUL WHEN SHARING THESE RESULTS - MAY CONTAIN YOUR SENSITIVE DATA - # logger.debug(loggable_response) + # logger.debug(self.log_response_safely(server_response)) if content_type == "application/xml": self.parent_srv._namespace.detect(server_response.content) diff --git a/tableauserverclient/server/endpoint/workbooks_endpoint.py b/tableauserverclient/server/endpoint/workbooks_endpoint.py index ada961af1..c008689e2 100644 --- a/tableauserverclient/server/endpoint/workbooks_endpoint.py +++ b/tableauserverclient/server/endpoint/workbooks_endpoint.py @@ -1000,7 +1000,8 @@ def publish( file_contents, connections=connections, ) - logger.debug(f"Request xml: {redact_xml(xml_request[:1000])} ") + if logger.isEnabledFor(logging.DEBUG): + logger.debug(f"Request xml: {redact_xml(xml_request[:1000])} ") # Send the publishing request to server try: diff --git a/test/test_endpoint.py b/test/test_endpoint.py index e4670d1fc..75515f1ea 100644 --- a/test/test_endpoint.py +++ b/test/test_endpoint.py @@ -1,3 +1,5 @@ +import logging +from unittest import mock from pathlib import Path import pytest import requests @@ -159,3 +161,13 @@ def test_server_response_error_missing_summary_and_detail(): assert err.code == "500001" assert err.summary == "" assert err.detail == "" + + +def test_response_body_not_redacted_when_not_logged(server: TSC.Server, caplog: pytest.LogCaptureFixture) -> None: + url = "http://test/" + endpoint = Endpoint(server) + with caplog.at_level(logging.DEBUG, logger="TSC"), requests_mock.mock() as m: + m.get(url, text="", headers={"Content-Type": "application/xml; charset=utf-8"}) + with mock.patch("tableauserverclient.helpers.strings.redact_xml") as redact: + endpoint.get_request(url) + redact.assert_not_called() diff --git a/test/test_regression_tests.py b/test/test_regression_tests.py index 21fbf3848..77b75af0c 100644 --- a/test/test_regression_tests.py +++ b/test/test_regression_tests.py @@ -82,3 +82,64 @@ def test_redact_password_really_not_xml(): assert redacted.find("my_s per_secre>_passphrase_which_nobody_should_ever_see") == -1 assert redacted.find("passphrase") == -1, redacted assert redacted.find("cookie") == -1, redacted + + +def test_redact_personal_access_token_secret(): + redacted = redact_xml( + b'' + b'' + ) + assert redacted.find(b"PAT-SECRET-VALUE") == -1, redacted + assert redacted.find(b"my-token") > 0, redacted + assert redacted.find(b"mysite") > 0, redacted + + +def test_redact_jwt(): + redacted = redact_xml( + '' + '' + ) + assert redacted.find("eyJhbGciOiJIUzI1NiJ9") == -1, redacted + assert redacted.find("isUat") > 0, redacted + + +def test_redact_session_token_in_namespaced_response(): + redacted = redact_xml( + '' + '' + ) + assert redacted.find("SESSION-TOKEN-VALUE") == -1, redacted + assert redacted.find("6b7179ba") > 0, redacted + assert redacted.find("1a96d216") > 0, redacted + + +def test_redact_unknown_credentials_attribute(): + redacted = redact_xml('') + assert redacted.find("UNKNOWN-SECRET") == -1, redacted + assert redacted.find('name="u"') > 0, redacted + + +def test_redact_client_secret(): + redacted = redact_xml( + '' + ) + assert redacted.find("CLIENT-SECRET") == -1, redacted + assert redacted.find('clientId="id"') > 0, redacted + + +def test_redact_truncated_xml_does_not_return_secret(): + body = ( + b'' + b'' + ) + redacted = redact_xml(body[:110]) + assert redacted.find(b"PAT-SECRET") == -1, redacted + + +def test_redact_unparseable_without_password_returns_placeholder(): + redacted = redact_xml( + "--boundary\r\nContent-Disposition: name=request_payload\r\n\r\n