diff --git a/python/private/pypi/parse_simpleapi_html.bzl b/python/private/pypi/parse_simpleapi_html.bzl index e919c8ca55..2b8ad8320c 100644 --- a/python/private/pypi/parse_simpleapi_html.bzl +++ b/python/private/pypi/parse_simpleapi_html.bzl @@ -13,7 +13,7 @@ # limitations under the License. """ -Parse SimpleAPI HTML in Starlark. +Parse SimpleAPI HTML and JSON (PEP 691) in Starlark. """ load("//python/private:normalize_name.bzl", "normalize_name") @@ -46,6 +46,11 @@ def parse_simpleapi_html(*, content, parse_index = False): then it will be `None`. An empty string yank reason means that the package is yanked but the reason is not provided. """ + # Detect PEP 691 JSON responses (some servers return JSON even when + # Accept: text/html is requested, ignoring content negotiation). + if content.lstrip()[:1] == "{": + return _parse_simpleapi_json(content = content, parse_index = parse_index) + sdists = {} whls = {} hashes_by_version = {} @@ -158,6 +163,94 @@ def parse_simpleapi_html(*, content, parse_index = False): hashes_by_version = hashes_by_version, ) +def _parse_simpleapi_json(*, content, parse_index): + """Parse a PEP 691 Simple API JSON response. + + Args: + content: {type}`str` The JSON content from the Simple API. + parse_index: {type}`bool` whether this is the root index page listing + all packages, or an individual package page. + + Returns: + Same structure as {obj}`parse_simpleapi_html`. + """ + data = json.decode(content) + + if parse_index: + packages = {} + for project in data.get("projects", []): + name = project.get("name", "") + if not name: + continue + normalized = normalize_name(name) + # PEP 691 index entries have no URL; construct a relative one. + url = project.get("url", "{}/".format(normalized)) + packages[normalized] = url + return packages + + sdists = {} + whls = {} + hashes_by_version = {} + + for file_info in data.get("files", []): + filename = file_info.get("filename", "") + if not filename: + continue + + url = file_info.get("url", "") + + # Build canonical : digests from all hashes and pick the + # preferred one (sha256 first) to use as the artifact key. + all_digests = [] + for algo, hex_digest in file_info.get("hashes", {}).items(): + d = hash.digest(algo, hex_digest) + if d: + all_digests.append(d) + digest = hash.preferred_digest(all_digests) + + # PEP 691: yanked is absent/false (not yanked), true (yanked, no + # reason), or a reason string. + yanked_val = file_info.get("yanked", None) + if yanked_val == False or yanked_val == None: + yanked = None + elif yanked_val == True: + yanked = "" + else: + yanked = str(yanked_val) + + # PEP 714: "core-metadata" or "dist-info-metadata" field. + metadata_sha256 = "" + metadata_url = "" + core_meta = file_info.get("core-metadata") or file_info.get("dist-info-metadata") + if core_meta: + if type(core_meta) == "dict": + metadata_sha256 = core_meta.get("sha256", "") + metadata_url = url + ".metadata" + + version = version_from_filename(filename) + hashes_by_version.setdefault(version, []).append(digest) + + dist = struct( + filename = filename, + version = version, + url = url, + digest = digest, + metadata_sha256 = metadata_sha256, + metadata_url = metadata_url, + yanked = yanked, + ) + + if filename.endswith(".whl"): + whls[digest] = dist + else: + sdists[digest] = dist + + return struct( + sdists = sdists, + whls = whls, + hashes_by_version = hashes_by_version, + ) + def _parse_attrs(attr_string): """Parses attributes from a pre-sliced string.""" attrs = {} diff --git a/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl index 975fcd0ee6..9b1ceb1d76 100644 --- a/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl +++ b/tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl @@ -358,6 +358,137 @@ def _test_sha256_fragment_digest(env): _tests.append(_test_sha256_fragment_digest) +def _test_json_sdist(env): + """PEP 691: JSON response for a package page with an sdist.""" + content = json.encode({ + "meta": {"api-version": "1.1"}, + "name": "foo", + "versions": ["0.0.1"], + "files": [ + { + "filename": "foo-0.0.1.tar.gz", + "url": "https://example.org/full-url/foo-0.0.1.tar.gz", + "hashes": {"sha256": "deadbeefasource"}, + "requires-python": ">=3.7", + "yanked": False, + }, + ], + }) + got = parse_simpleapi_html(content = content) + + env.expect.that_collection(got.sdists).has_size(1) + env.expect.that_collection(got.whls).has_size(0) + env.expect.that_dict(got.hashes_by_version).contains_exactly({ + "0.0.1": ["sha256:deadbeefasource"], + }) + + actual = env.expect.that_struct( + got.sdists["sha256:deadbeefasource"], + attrs = dict( + filename = subjects.str, + digest = subjects.str, + url = subjects.str, + yanked = subjects.str, + version = subjects.str, + ), + ) + actual.filename().equals("foo-0.0.1.tar.gz") + actual.digest().equals("sha256:deadbeefasource") + actual.url().equals("https://example.org/full-url/foo-0.0.1.tar.gz") + actual.yanked().equals(None) + actual.version().equals("0.0.1") + +_tests.append(_test_json_sdist) + +def _test_json_whl_with_metadata(env): + """PEP 691: JSON response for a whl with core-metadata.""" + content = json.encode({ + "meta": {"api-version": "1.1"}, + "name": "foo", + "versions": ["0.0.2"], + "files": [ + { + "filename": "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "url": "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "hashes": {"sha256": "deadbeef"}, + "core-metadata": {"sha256": "deadb00f"}, + "yanked": False, + }, + ], + }) + got = parse_simpleapi_html(content = content) + + env.expect.that_collection(got.whls).has_size(1) + env.expect.that_collection(got.sdists).has_size(0) + + actual = env.expect.that_struct( + got.whls["sha256:deadbeef"], + attrs = dict( + filename = subjects.str, + metadata_sha256 = subjects.str, + metadata_url = subjects.str, + digest = subjects.str, + url = subjects.str, + yanked = subjects.str, + version = subjects.str, + ), + ) + actual.filename().equals("foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") + actual.metadata_sha256().equals("deadb00f") + actual.metadata_url().equals("https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata") + actual.digest().equals("sha256:deadbeef") + actual.url().equals("https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") + actual.yanked().equals(None) + actual.version().equals("0.0.2") + +_tests.append(_test_json_whl_with_metadata) + +def _test_json_yanked(env): + """PEP 691: JSON response with yanked entries.""" + content = json.encode({ + "meta": {"api-version": "1.1"}, + "name": "foo", + "versions": ["0.0.1"], + "files": [ + { + "filename": "foo-0.0.1.tar.gz", + "url": "https://example.org/full-url/foo-0.0.1.tar.gz", + "hashes": {"sha256": "deadbeef01"}, + "yanked": True, + }, + { + "filename": "foo-0.0.1-py3-none-any.whl", + "url": "https://example.org/full-url/foo-0.0.1-py3-none-any.whl", + "hashes": {"sha256": "deadbeef02"}, + "yanked": "security issue", + }, + ], + }) + got = parse_simpleapi_html(content = content) + + env.expect.that_str(got.sdists["sha256:deadbeef01"].yanked).equals("") + env.expect.that_str(got.whls["sha256:deadbeef02"].yanked).equals("security issue") + +_tests.append(_test_json_yanked) + +def _test_json_index(env): + """PEP 691: JSON response for the root index page.""" + content = json.encode({ + "meta": {"api-version": "1.0"}, + "projects": [ + {"name": "foo"}, + {"name": "B-Ar"}, + ], + }) + got = parse_simpleapi_html(content = content, parse_index = True) + + env.expect.that_dict(got).contains_exactly({ + "foo": "foo/", + "b_ar": "b_ar/", + }) + +_tests.append(_test_json_index) + def parse_simpleapi_html_test_suite(name): """Create the test suite.