Skip to content

Commit

Permalink
Read package metadata from pyproject.toml when statically defined (#…
Browse files Browse the repository at this point in the history
…2676)

## Summary

Now that we're resolving metadata more aggressively for local sources,
it's worth doing this. We now pull metadata from the `pyproject.toml`
directly if it's statically-defined.

Closes #2629.
  • Loading branch information
charliermarsh authored Mar 27, 2024
1 parent 248d6f8 commit 365c292
Show file tree
Hide file tree
Showing 9 changed files with 302 additions and 34 deletions.
2 changes: 2 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

54 changes: 53 additions & 1 deletion crates/pep508-rs/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -471,6 +471,32 @@ impl Requirement {
(true, Vec::new())
}
}

/// Return the requirement with an additional marker added, to require the given extra.
///
/// For example, given `flask >= 2.0.2`, calling `with_extra_marker("dotenv")` would return
/// `flask >= 2.0.2 ; extra == "dotenv"`.
pub fn with_extra_marker(self, extra: &ExtraName) -> Self {
let marker = match self.marker {
Some(expression) => MarkerTree::And(vec![
expression,
MarkerTree::Expression(MarkerExpression {
l_value: MarkerValue::Extra,
operator: MarkerOperator::Equal,
r_value: MarkerValue::QuotedString(extra.to_string()),
}),
]),
None => MarkerTree::Expression(MarkerExpression {
l_value: MarkerValue::Extra,
operator: MarkerOperator::Equal,
r_value: MarkerValue::QuotedString(extra.to_string()),
}),
};
Self {
marker: Some(marker),
..self
}
}
}

impl UnnamedRequirement {
Expand Down Expand Up @@ -1560,7 +1586,7 @@ mod tests {
use insta::assert_snapshot;

use pep440_rs::{Operator, Version, VersionPattern, VersionSpecifier};
use uv_normalize::{ExtraName, PackageName};
use uv_normalize::{ExtraName, InvalidNameError, PackageName};

use crate::marker::{
parse_markers_impl, MarkerExpression, MarkerOperator, MarkerTree, MarkerValue,
Expand Down Expand Up @@ -2264,4 +2290,30 @@ mod tests {

Ok(())
}

#[test]
fn add_extra_marker() -> Result<(), InvalidNameError> {
let requirement = Requirement::from_str("pytest").unwrap();
let expected = Requirement::from_str("pytest; extra == 'dotenv'").unwrap();
let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?);
assert_eq!(actual, expected);

let requirement = Requirement::from_str("pytest; '4.0' >= python_version").unwrap();
let expected =
Requirement::from_str("pytest; '4.0' >= python_version and extra == 'dotenv'").unwrap();
let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?);
assert_eq!(actual, expected);

let requirement =
Requirement::from_str("pytest; '4.0' >= python_version or sys_platform == 'win32'")
.unwrap();
let expected = Requirement::from_str(
"pytest; ('4.0' >= python_version or sys_platform == 'win32') and extra == 'dotenv'",
)
.unwrap();
let actual = requirement.with_extra_marker(&ExtraName::from_str("dotenv")?);
assert_eq!(actual, expected);

Ok(())
}
}
2 changes: 2 additions & 0 deletions crates/pypi-types/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -18,12 +18,14 @@ pep508_rs = { workspace = true, features = ["rkyv", "serde"] }
uv-normalize = { workspace = true }

chrono = { workspace = true, features = ["serde"] }
indexmap = { workspace = true, features = ["serde"] }
mailparse = { workspace = true }
once_cell = { workspace = true }
regex = { workspace = true }
rkyv = { workspace = true }
serde = { workspace = true }
thiserror = { workspace = true }
toml = { workspace = true }
tracing = { workspace = true }
url = { workspace = true }

Expand Down
187 changes: 174 additions & 13 deletions crates/pypi-types/src/metadata.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Derived from `pypi_types_crate`.

use indexmap::IndexMap;
use std::io;
use std::str::FromStr;

Expand All @@ -22,11 +23,10 @@ use crate::LenientVersionSpecifiers;
/// fields that are relevant to dependency resolution.
///
/// At present, we support up to version 2.3 of the metadata specification.
#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)]
#[derive(Serialize, Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
pub struct Metadata23 {
// Mandatory fields
pub metadata_version: String,
pub name: PackageName,
pub version: Version,
// Optional fields
Expand All @@ -46,6 +46,9 @@ pub enum Error {
/// mail parse error
#[error(transparent)]
MailParse(#[from] MailParseError),
/// TOML parse error
#[error(transparent)]
Toml(#[from] toml::de::Error),
/// Metadata field not found
#[error("metadata field {0} not found")]
FieldNotFound(&'static str),
Expand Down Expand Up @@ -86,9 +89,6 @@ impl Metadata23 {
pub fn parse_metadata(content: &[u8]) -> Result<Self, Error> {
let headers = Headers::parse(content)?;

let metadata_version = headers
.get_first_value("Metadata-Version")
.ok_or(Error::FieldNotFound("Metadata-Version"))?;
let name = PackageName::new(
headers
.get_first_value("Name")
Expand Down Expand Up @@ -124,7 +124,6 @@ impl Metadata23 {
.collect::<Vec<_>>();

Ok(Self {
metadata_version,
name,
version,
requires_dist,
Expand Down Expand Up @@ -200,22 +199,109 @@ impl Metadata23 {
.collect::<Vec<_>>();

Ok(Self {
metadata_version,
name,
version,
requires_dist,
requires_python,
provides_extras,
})
}

/// Extract the metadata from a `pyproject.toml` file, as specified in PEP 621.
pub fn parse_pyproject_toml(contents: &str) -> Result<Self, Error> {
let pyproject_toml: PyProjectToml = toml::from_str(contents)?;

let project = pyproject_toml
.project
.ok_or(Error::FieldNotFound("project"))?;

// If any of the fields we need were declared as dynamic, we can't use the `pyproject.toml` file.
let dynamic = project.dynamic.unwrap_or_default();
for field in dynamic {
match field.as_str() {
"dependencies" => return Err(Error::DynamicField("dependencies")),
"optional-dependencies" => {
return Err(Error::DynamicField("optional-dependencies"))
}
"requires-python" => return Err(Error::DynamicField("requires-python")),
"version" => return Err(Error::DynamicField("version")),
_ => (),
}
}

let name = project.name;
let version = project.version.ok_or(Error::FieldNotFound("version"))?;
let requires_python = project.requires_python.map(VersionSpecifiers::from);

// Extract the requirements.
let mut requires_dist = project
.dependencies
.unwrap_or_default()
.into_iter()
.map(Requirement::from)
.collect::<Vec<_>>();

// Extract the optional dependencies.
let mut provides_extras: Vec<ExtraName> = Vec::new();
for (extra, requirements) in project.optional_dependencies.unwrap_or_default() {
requires_dist.extend(
requirements
.into_iter()
.map(Requirement::from)
.map(|requirement| requirement.with_extra_marker(&extra))
.collect::<Vec<_>>(),
);
provides_extras.push(extra);
}

Ok(Self {
name,
version,
requires_dist,
requires_python,
provides_extras,
})
}
}

/// A `pyproject.toml` as specified in PEP 517.
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
pub(crate) struct PyProjectToml {
/// Project metadata
pub(crate) project: Option<Project>,
}

/// PEP 621 project metadata.
///
/// This is a subset of the full metadata specification, and only includes the fields that are
/// relevant for dependency resolution.
///
/// See <https://packaging.python.org/en/latest/specifications/pyproject-toml>.
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
pub(crate) struct Project {
/// The name of the project
pub(crate) name: PackageName,
/// The version of the project as supported by PEP 440
pub(crate) version: Option<Version>,
/// The Python version requirements of the project
pub(crate) requires_python: Option<LenientVersionSpecifiers>,
/// Project dependencies
pub(crate) dependencies: Option<Vec<LenientRequirement>>,
/// Optional dependencies
pub(crate) optional_dependencies: Option<IndexMap<ExtraName, Vec<LenientRequirement>>>,
/// Specifies which fields listed by PEP 621 were intentionally unspecified
/// so another tool can/will provide such metadata dynamically.
pub(crate) dynamic: Option<Vec<String>>,
}

/// Python Package Metadata 1.0 and later as specified in
/// <https://peps.python.org/pep-0241/>.
///
/// This is a subset of the full metadata specification, and only includes the
/// fields that have been consistent across all versions of the specification.
#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)]
#[derive(Deserialize, Debug, Clone)]
#[serde(rename_all = "kebab-case")]
pub struct Metadata10 {
pub name: PackageName,
Expand Down Expand Up @@ -303,19 +389,16 @@ mod tests {

let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0";
let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap();
assert_eq!(meta.metadata_version, "1.0");
assert_eq!(meta.name, PackageName::from_str("asdf").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));

let s = "Metadata-Version: 1.0\nName: asdf\nVersion: 1.0\nAuthor: 中文\n\n一个 Python 包";
let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap();
assert_eq!(meta.metadata_version, "1.0");
assert_eq!(meta.name, PackageName::from_str("asdf").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));

let s = "Metadata-Version: 1.0\nName: =?utf-8?q?foobar?=\nVersion: 1.0";
let meta = Metadata23::parse_metadata(s.as_bytes()).unwrap();
assert_eq!(meta.metadata_version, "1.0");
assert_eq!(meta.name, PackageName::from_str("foobar").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));

Expand All @@ -340,7 +423,6 @@ mod tests {

let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0";
let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap();
assert_eq!(meta.metadata_version, "2.3");
assert_eq!(meta.name, PackageName::from_str("asdf").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));

Expand All @@ -350,9 +432,88 @@ mod tests {

let s = "Metadata-Version: 2.3\nName: asdf\nVersion: 1.0\nRequires-Dist: foo";
let meta = Metadata23::parse_pkg_info(s.as_bytes()).unwrap();
assert_eq!(meta.metadata_version, "2.3");
assert_eq!(meta.name, PackageName::from_str("asdf").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));
assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]);
}

#[test]
fn test_parse_pyproject_toml() {
let s = r#"
[project]
name = "asdf"
"#;
let meta = Metadata23::parse_pyproject_toml(s);
assert!(matches!(meta, Err(Error::FieldNotFound("version"))));

let s = r#"
[project]
name = "asdf"
dynamic = ["version"]
"#;
let meta = Metadata23::parse_pyproject_toml(s);
assert!(matches!(meta, Err(Error::DynamicField("version"))));

let s = r#"
[project]
name = "asdf"
version = "1.0"
"#;
let meta = Metadata23::parse_pyproject_toml(s).unwrap();
assert_eq!(meta.name, PackageName::from_str("asdf").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));
assert!(meta.requires_python.is_none());
assert!(meta.requires_dist.is_empty());
assert!(meta.provides_extras.is_empty());

let s = r#"
[project]
name = "asdf"
version = "1.0"
requires-python = ">=3.6"
"#;
let meta = Metadata23::parse_pyproject_toml(s).unwrap();
assert_eq!(meta.name, PackageName::from_str("asdf").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));
assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap()));
assert!(meta.requires_dist.is_empty());
assert!(meta.provides_extras.is_empty());

let s = r#"
[project]
name = "asdf"
version = "1.0"
requires-python = ">=3.6"
dependencies = ["foo"]
"#;
let meta = Metadata23::parse_pyproject_toml(s).unwrap();
assert_eq!(meta.name, PackageName::from_str("asdf").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));
assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap()));
assert_eq!(meta.requires_dist, vec!["foo".parse().unwrap()]);
assert!(meta.provides_extras.is_empty());

let s = r#"
[project]
name = "asdf"
version = "1.0"
requires-python = ">=3.6"
dependencies = ["foo"]
[project.optional-dependencies]
dotenv = ["bar"]
"#;
let meta = Metadata23::parse_pyproject_toml(s).unwrap();
assert_eq!(meta.name, PackageName::from_str("asdf").unwrap());
assert_eq!(meta.version, Version::new([1, 0]));
assert_eq!(meta.requires_python, Some(">=3.6".parse().unwrap()));
assert_eq!(
meta.requires_dist,
vec![
"foo".parse().unwrap(),
"bar; extra == \"dotenv\"".parse().unwrap()
]
);
assert_eq!(meta.provides_extras, vec!["dotenv".parse().unwrap()]);
}
}
Loading

0 comments on commit 365c292

Please sign in to comment.