Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
95 changes: 94 additions & 1 deletion python/private/pypi/parse_simpleapi_html.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.

"""
Parse SimpleAPI HTML in Starlark.
Parse SimpleAPI HTML and JSON (PEP 691) in Starlark.
"""

load("//python/private:normalize_name.bzl", "normalize_name")
Expand Down Expand Up @@ -46,6 +46,11 @@ def parse_simpleapi_html(*, content, parse_index = False):
then it will be `None`. An empty string yank reason means that the package is yanked but
the reason is not provided.
"""
# Detect PEP 691 JSON responses (some servers return JSON even when
# Accept: text/html is requested, ignoring content negotiation).
if content.lstrip()[:1] == "{":
return _parse_simpleapi_json(content = content, parse_index = parse_index)

sdists = {}
whls = {}
hashes_by_version = {}
Expand Down Expand Up @@ -158,6 +163,94 @@ def parse_simpleapi_html(*, content, parse_index = False):
hashes_by_version = hashes_by_version,
)

def _parse_simpleapi_json(*, content, parse_index):
"""Parse a PEP 691 Simple API JSON response.

Args:
content: {type}`str` The JSON content from the Simple API.
parse_index: {type}`bool` whether this is the root index page listing
all packages, or an individual package page.

Returns:
Same structure as {obj}`parse_simpleapi_html`.
"""
data = json.decode(content)

if parse_index:
packages = {}
for project in data.get("projects", []):
name = project.get("name", "")
if not name:
continue
normalized = normalize_name(name)
# PEP 691 index entries have no URL; construct a relative one.
url = project.get("url", "{}/".format(normalized))
packages[normalized] = url
return packages

sdists = {}
whls = {}
hashes_by_version = {}

for file_info in data.get("files", []):
filename = file_info.get("filename", "")
if not filename:
continue

url = file_info.get("url", "")

# Build canonical <algo>:<hex> digests from all hashes and pick the
# preferred one (sha256 first) to use as the artifact key.
all_digests = []
for algo, hex_digest in file_info.get("hashes", {}).items():
d = hash.digest(algo, hex_digest)
if d:
all_digests.append(d)
digest = hash.preferred_digest(all_digests)

# PEP 691: yanked is absent/false (not yanked), true (yanked, no
# reason), or a reason string.
yanked_val = file_info.get("yanked", None)
if yanked_val == False or yanked_val == None:
yanked = None
elif yanked_val == True:
yanked = ""
else:
yanked = str(yanked_val)

# PEP 714: "core-metadata" or "dist-info-metadata" field.
metadata_sha256 = ""
metadata_url = ""
core_meta = file_info.get("core-metadata") or file_info.get("dist-info-metadata")
if core_meta:
if type(core_meta) == "dict":
metadata_sha256 = core_meta.get("sha256", "")
metadata_url = url + ".metadata"

version = version_from_filename(filename)
hashes_by_version.setdefault(version, []).append(digest)

dist = struct(
filename = filename,
version = version,
url = url,
digest = digest,
metadata_sha256 = metadata_sha256,
metadata_url = metadata_url,
yanked = yanked,
)

if filename.endswith(".whl"):
whls[digest] = dist
else:
sdists[digest] = dist

return struct(
sdists = sdists,
whls = whls,
hashes_by_version = hashes_by_version,
)

def _parse_attrs(attr_string):
"""Parses attributes from a pre-sliced string."""
attrs = {}
Expand Down
131 changes: 131 additions & 0 deletions tests/pypi/parse_simpleapi_html/parse_simpleapi_html_tests.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -358,6 +358,137 @@ def _test_sha256_fragment_digest(env):

_tests.append(_test_sha256_fragment_digest)

def _test_json_sdist(env):
"""PEP 691: JSON response for a package page with an sdist."""
content = json.encode({
"meta": {"api-version": "1.1"},
"name": "foo",
"versions": ["0.0.1"],
"files": [
{
"filename": "foo-0.0.1.tar.gz",
"url": "https://example.org/full-url/foo-0.0.1.tar.gz",
"hashes": {"sha256": "deadbeefasource"},
"requires-python": ">=3.7",
"yanked": False,
},
],
})
got = parse_simpleapi_html(content = content)

env.expect.that_collection(got.sdists).has_size(1)
env.expect.that_collection(got.whls).has_size(0)
env.expect.that_dict(got.hashes_by_version).contains_exactly({
"0.0.1": ["sha256:deadbeefasource"],
})

actual = env.expect.that_struct(
got.sdists["sha256:deadbeefasource"],
attrs = dict(
filename = subjects.str,
digest = subjects.str,
url = subjects.str,
yanked = subjects.str,
version = subjects.str,
),
)
actual.filename().equals("foo-0.0.1.tar.gz")
actual.digest().equals("sha256:deadbeefasource")
actual.url().equals("https://example.org/full-url/foo-0.0.1.tar.gz")
actual.yanked().equals(None)
actual.version().equals("0.0.1")

_tests.append(_test_json_sdist)

def _test_json_whl_with_metadata(env):
"""PEP 691: JSON response for a whl with core-metadata."""
content = json.encode({
"meta": {"api-version": "1.1"},
"name": "foo",
"versions": ["0.0.2"],
"files": [
{
"filename": "foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"url": "https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl",
"hashes": {"sha256": "deadbeef"},
"core-metadata": {"sha256": "deadb00f"},
"yanked": False,
},
],
})
got = parse_simpleapi_html(content = content)

env.expect.that_collection(got.whls).has_size(1)
env.expect.that_collection(got.sdists).has_size(0)

actual = env.expect.that_struct(
got.whls["sha256:deadbeef"],
attrs = dict(
filename = subjects.str,
metadata_sha256 = subjects.str,
metadata_url = subjects.str,
digest = subjects.str,
url = subjects.str,
yanked = subjects.str,
version = subjects.str,
),
)
actual.filename().equals("foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
actual.metadata_sha256().equals("deadb00f")
actual.metadata_url().equals("https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata")
actual.digest().equals("sha256:deadbeef")
actual.url().equals("https://example.org/full-url/foo-0.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
actual.yanked().equals(None)
actual.version().equals("0.0.2")

_tests.append(_test_json_whl_with_metadata)

def _test_json_yanked(env):
"""PEP 691: JSON response with yanked entries."""
content = json.encode({
"meta": {"api-version": "1.1"},
"name": "foo",
"versions": ["0.0.1"],
"files": [
{
"filename": "foo-0.0.1.tar.gz",
"url": "https://example.org/full-url/foo-0.0.1.tar.gz",
"hashes": {"sha256": "deadbeef01"},
"yanked": True,
},
{
"filename": "foo-0.0.1-py3-none-any.whl",
"url": "https://example.org/full-url/foo-0.0.1-py3-none-any.whl",
"hashes": {"sha256": "deadbeef02"},
"yanked": "security issue",
},
],
})
got = parse_simpleapi_html(content = content)

env.expect.that_str(got.sdists["sha256:deadbeef01"].yanked).equals("")
env.expect.that_str(got.whls["sha256:deadbeef02"].yanked).equals("security issue")

_tests.append(_test_json_yanked)

def _test_json_index(env):
"""PEP 691: JSON response for the root index page."""
content = json.encode({
"meta": {"api-version": "1.0"},
"projects": [
{"name": "foo"},
{"name": "B-Ar"},
],
})
got = parse_simpleapi_html(content = content, parse_index = True)

env.expect.that_dict(got).contains_exactly({
"foo": "foo/",
"b_ar": "b_ar/",
})

_tests.append(_test_json_index)

def parse_simpleapi_html_test_suite(name):
"""Create the test suite.

Expand Down
Loading