doiget_core/
pubmed.rs

1//! PubMed identifiers (#500, ADR-0061): a PMID or PMCID is resolved to the
2//! DOI PubMed lists for it, and the paper is then fetched, cited and stored
3//! under that DOI like any other.
4//!
5//! A PubMed-identified entry used to be reported and left: "identified only
6//! by PMID, which doiget cannot resolve yet". It is now an alias. The store
7//! identity stays the DOI, so no safekey class, no `Ref` variant and no
8//! store layout change: the id is looked up once, through NCBI E-utilities
9//! (`esummary.fcgi`, `db=pubmed` for a PMID, `db=pmc` for a PMCID), and the
10//! `articleids[]` entry with `idtype == "doi"` is the answer. A record with
11//! no DOI there is reported as such, never guessed from.
12//!
13//! NCBI's usage guidance (NBK25497): at most 3 requests a second without an
14//! API key, which is tighter than doiget's global cap and therefore a
15//! [`crate::SOURCE_RATE_OVERRIDES`] entry; `tool` and `email` identify the
16//! caller and are sent (the email only when one is configured). doiget sends
17//! no API key, so the 10-a-second keyed rate is never assumed.
18
19use serde_json::Value;
20use url::Url;
21
22use crate::provenance::{Capability, LogEvent, LogResult, RowInput};
23use crate::refs::{ParseError, ParsedEntry};
24use crate::source::{FetchContext, FetchError};
25use crate::{Doi, Ref};
26
27/// HTTP and rate-limiter source key for NCBI E-utilities.
28pub const NCBI: &str = "ncbi";
29/// Base-URL override for [`NCBI`].
30pub const NCBI_BASE_ENV: &str = "DOIGET_NCBI_BASE";
31const NCBI_DEFAULT: &str = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/";
32
33/// A PubMed identifier.
34#[derive(Debug, Clone, PartialEq, Eq)]
35pub enum PubmedId {
36    /// A PubMed id.
37    Pmid(Digits),
38    /// A PubMed Central id (the `PMC` prefix removed).
39    Pmcid(Digits),
40}
41
42/// The numeric part of a PubMed id: 1 to 12 ASCII digits. Only
43/// [`PubmedId::parse`] and [`PubmedId::from_kind`] make one, so a
44/// `PubmedId` always holds a valid id (#649 review).
45#[derive(Debug, Clone, PartialEq, Eq)]
46pub struct Digits(String);
47
48impl Digits {
49    /// The digits.
50    #[must_use]
51    pub fn as_str(&self) -> &str {
52        &self.0
53    }
54}
55
56impl std::fmt::Display for Digits {
57    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58        f.write_str(&self.0)
59    }
60}
61
62impl PubmedId {
63    /// Parse `pmid:9659853`, `PMID: 9659853`, `pmcid:PMC3531190`,
64    /// `PMC3531190`, or a `pubmed.ncbi.nlm.nih.gov/<pmid>` /
65    /// `…/pmc/articles/PMC<id>` URL. Bare digits are not a PMID: they are too
66    /// easily something else.
67    #[must_use]
68    pub fn parse(input: &str) -> Option<Self> {
69        let s = input.trim();
70        let lower = s.to_ascii_lowercase();
71        if let Some(rest) = lower.strip_prefix("pmid:") {
72            return digits(rest.trim()).map(Self::Pmid);
73        }
74        if let Some(rest) = lower.strip_prefix("pmcid:") {
75            let rest = rest.trim();
76            return digits(rest.strip_prefix("pmc").unwrap_or(rest)).map(Self::Pmcid);
77        }
78        if let Some(rest) = lower.strip_prefix("pmc") {
79            return digits(rest).map(Self::Pmcid);
80        }
81        let url = Url::parse(s).ok()?;
82        let segs: Vec<&str> = url.path_segments()?.filter(|p| !p.is_empty()).collect();
83        match (url.host_str()?, segs.as_slice()) {
84            ("pubmed.ncbi.nlm.nih.gov", [id]) => digits(id).map(Self::Pmid),
85            ("www.ncbi.nlm.nih.gov" | "ncbi.nlm.nih.gov", ["pmc", "articles", id]) => {
86                let id = id.to_ascii_lowercase();
87                digits(id.strip_prefix("pmc")?).map(Self::Pmcid)
88            }
89            ("pmc.ncbi.nlm.nih.gov", ["articles", id]) => {
90                let id = id.to_ascii_lowercase();
91                digits(id.strip_prefix("pmc")?).map(Self::Pmcid)
92            }
93            _ => None,
94        }
95    }
96
97    /// From a bibliography's `UnsupportedIdentifier { kind, value }`.
98    #[must_use]
99    pub fn from_kind(kind: &str, value: &str) -> Option<Self> {
100        match kind {
101            "PMID" => digits(value.trim()).map(Self::Pmid),
102            "PMCID" => {
103                let v = value.trim().to_ascii_lowercase();
104                digits(v.strip_prefix("pmc").unwrap_or(&v)).map(Self::Pmcid)
105            }
106            _ => None,
107        }
108    }
109
110    /// `PMID 9659853` / `PMCID PMC3531190`.
111    #[must_use]
112    pub fn display(&self) -> String {
113        match self {
114            Self::Pmid(id) => format!("PMID {id}"),
115            Self::Pmcid(id) => format!("PMCID PMC{id}"),
116        }
117    }
118
119    fn db_and_id(&self) -> (&'static str, &str) {
120        match self {
121            Self::Pmid(id) => ("pubmed", id.as_str()),
122            Self::Pmcid(id) => ("pmc", id.as_str()),
123        }
124    }
125}
126
127fn digits(s: &str) -> Option<Digits> {
128    (!s.is_empty() && s.len() <= 12 && s.chars().all(|c| c.is_ascii_digit()))
129        .then(|| Digits(s.to_string()))
130}
131
132/// What PubMed says about an id.
133#[derive(Debug, Clone, PartialEq, Eq)]
134pub enum Lookup {
135    /// The record lists this DOI.
136    Doi(Doi),
137    /// The record exists and lists no DOI.
138    NoDoi,
139    /// PubMed has no record under this id.
140    NoRecord,
141}
142
143impl Lookup {
144    /// A sentence for the user when there is no DOI to go on.
145    #[must_use]
146    pub fn reason(&self, id: &PubmedId) -> Option<String> {
147        match self {
148            Self::Doi(_) => None,
149            Self::NoDoi => Some(format!(
150                "PubMed's record for {} lists no DOI, and doiget reaches PubMed records through their DOI",
151                id.display()
152            )),
153            Self::NoRecord => Some(format!("PubMed has no record for {}", id.display())),
154        }
155    }
156}
157
158/// Ask NCBI E-utilities for the DOI of `id`.
159///
160/// # Errors
161///
162/// [`FetchError`] for a transport failure or an answer that is not
163/// E-utilities JSON; a record without a DOI, or no record, is a [`Lookup`].
164pub async fn lookup(id: &PubmedId, ctx: &FetchContext) -> Result<Lookup, FetchError> {
165    let (db, uid) = id.db_and_id();
166    let raw_base = std::env::var(NCBI_BASE_ENV).unwrap_or_else(|_| NCBI_DEFAULT.to_string());
167    let mut base = Url::parse(&raw_base).map_err(|e| FetchError::SourceSchema {
168        hint: format!("{NCBI_BASE_ENV}={raw_base:?} is not a URL: {e}"),
169    })?;
170    if !base.path().ends_with('/') {
171        base.set_path(&format!("{}/", base.path()));
172    }
173    let mut url = base
174        .join("esummary.fcgi")
175        .map_err(|e| FetchError::SourceSchema {
176            hint: format!("building the E-utilities URL: {e}"),
177        })?;
178    {
179        let mut q = url.query_pairs_mut();
180        q.append_pair("db", db)
181            .append_pair("id", uid)
182            .append_pair("retmode", "json")
183            .append_pair("tool", "doiget");
184        if let Some(email) = crate::orchestrator::configured_contact_email() {
185            q.append_pair("email", &email);
186        }
187    }
188
189    let _permit = ctx.rate_limiter.acquire(NCBI).await;
190    let shown = id.display();
191    let row = |result, size, error_code| RowInput {
192        event: LogEvent::Resolve,
193        result,
194        capability: Capability::Metadata,
195        ref_: Some(shown.as_str()),
196        source: Some(NCBI),
197        error_code,
198        size_bytes: size,
199        license: None,
200        store_path: None,
201        canonical_digest: None,
202    };
203    let body = match ctx.http.fetch_bytes(NCBI, url).await {
204        Ok((body, _)) => body,
205        Err(e) => {
206            let e = FetchError::Http(e);
207            let code = crate::ErrorCode::from(&e);
208            ctx.log
209                .append(row(LogResult::Err, None, Some(code.as_wire())))?;
210            return Err(e);
211        }
212    };
213    let v: Value = serde_json::from_slice(&body).map_err(|e| FetchError::SourceSchema {
214        hint: format!("E-utilities returned non-JSON for {shown}: {e}"),
215    })?;
216    let found = classify(&v, uid);
217    ctx.log.append(row(
218        LogResult::Ok,
219        Some(body.len() as u64),
220        matches!(found, Lookup::NoRecord).then_some("NOT_FOUND"),
221    ))?;
222    Ok(found)
223}
224
225/// Read an ESummary JSON answer for `uid`.
226fn classify(v: &Value, uid: &str) -> Lookup {
227    let Some(rec) = v.pointer("/result").and_then(|r| r.get(uid)) else {
228        return Lookup::NoRecord;
229    };
230    if rec.get("error").is_some() {
231        return Lookup::NoRecord;
232    }
233    rec.get("articleids")
234        .and_then(Value::as_array)
235        .and_then(|ids| {
236            ids.iter()
237                .find(|i| i.get("idtype").and_then(Value::as_str) == Some("doi"))
238        })
239        .and_then(|i| i.get("value").and_then(Value::as_str))
240        // Trimmed: `Doi::parse` refuses whitespace, and a DOI PubMed lists
241        // with a stray space must not read as "no DOI".
242        .and_then(|d| Doi::parse(d.trim()).ok())
243        .map_or(Lookup::NoDoi, Lookup::Doi)
244}
245
246/// A bibliography entry PubMed could not turn into a DOI.
247#[derive(Debug, Clone)]
248pub struct Unresolved {
249    /// The id as the entry named it.
250    pub id: PubmedId,
251    /// The citation key.
252    pub entry_key: Option<String>,
253    /// Why, for the user.
254    pub reason: String,
255    /// The code a surface reports it under: `NOT_FOUND` when PubMed has no
256    /// record, `NOT_IMPLEMENTED` when the record has no DOI, or the
257    /// transport error's.
258    pub code: crate::ErrorCode,
259}
260
261/// One bibliography entry after PubMed resolution.
262#[derive(Debug)]
263pub enum Resolved {
264    /// As parsed, or a PubMed id turned into its DOI.
265    Entry(Result<ParsedEntry, ParseError>),
266    /// A PubMed id with no DOI to go on.
267    Unresolved(Unresolved),
268}
269
270/// Resolve every PMID / PMCID entry of a parsed bibliography to its DOI,
271/// leaving every other entry as it was. One request per PubMed id, at
272/// NCBI's rate.
273///
274/// # Errors
275///
276/// A provenance-log failure (fail-closed); every other failure is an
277/// [`Unresolved`] entry.
278pub async fn resolve_entries(
279    entries: Vec<Result<ParsedEntry, ParseError>>,
280    ctx: &FetchContext,
281) -> Result<Vec<Resolved>, FetchError> {
282    let mut out = Vec::with_capacity(entries.len());
283    for entry in entries {
284        let (id, entry_key) = match &entry {
285            Err(ParseError::UnsupportedIdentifier {
286                kind,
287                value,
288                entry_key,
289            }) => match PubmedId::from_kind(kind, value) {
290                Some(id) => (id, entry_key.clone()),
291                None => {
292                    out.push(Resolved::Entry(entry));
293                    continue;
294                }
295            },
296            _ => {
297                out.push(Resolved::Entry(entry));
298                continue;
299            }
300        };
301        out.push(match lookup(&id, ctx).await {
302            Ok(Lookup::Doi(doi)) => Resolved::Entry(Ok(ParsedEntry {
303                ref_: Ref::Doi(doi),
304                entry_key,
305            })),
306            Ok(found) => Resolved::Unresolved(Unresolved {
307                reason: found.reason(&id).unwrap_or_default(),
308                code: if found == Lookup::NoRecord {
309                    crate::ErrorCode::NotFound
310                } else {
311                    crate::ErrorCode::NotImplemented
312                },
313                id,
314                entry_key,
315            }),
316            Err(FetchError::Log(e)) => return Err(FetchError::Log(e)),
317            Err(e) => Resolved::Unresolved(Unresolved {
318                reason: format!("looking up {} at NCBI failed: {e}", id.display()),
319                code: crate::ErrorCode::from(&e),
320                id,
321                entry_key,
322            }),
323        });
324    }
325    Ok(out)
326}
327
328/// [`lookup`] as its own logged session: `session_start` before and
329/// `session_end` after, for a caller whose context exists only for the
330/// lookup -- so its `resolve` row is bracketed like every other call's.
331///
332/// # Errors
333///
334/// As [`lookup`], plus a provenance-log failure on either bookend.
335pub async fn lookup_in_session(id: &PubmedId, ctx: &FetchContext) -> Result<Lookup, FetchError> {
336    bookend(ctx, LogEvent::SessionStart, LogResult::Ok)?;
337    let found = lookup(id, ctx).await;
338    bookend(
339        ctx,
340        LogEvent::SessionEnd,
341        if found.is_ok() {
342            LogResult::Ok
343        } else {
344            LogResult::Err
345        },
346    )?;
347    found
348}
349
350/// [`resolve_entries`] as its own logged session (see [`lookup_in_session`]).
351///
352/// # Errors
353///
354/// As [`resolve_entries`], plus a provenance-log failure on either bookend.
355pub async fn resolve_entries_in_session(
356    entries: Vec<Result<ParsedEntry, ParseError>>,
357    ctx: &FetchContext,
358) -> Result<Vec<Resolved>, FetchError> {
359    // No PubMed id, no lookup session: a bibliography without one must log
360    // exactly what it did before (#634 review -- one session_start per
361    // batch).
362    let any = entries.iter().any(|e| {
363        matches!(e, Err(ParseError::UnsupportedIdentifier { kind, value, .. })
364            if PubmedId::from_kind(kind, value).is_some())
365    });
366    if !any {
367        return Ok(entries.into_iter().map(Resolved::Entry).collect());
368    }
369    bookend(ctx, LogEvent::SessionStart, LogResult::Ok)?;
370    let resolved = resolve_entries(entries, ctx).await;
371    bookend(
372        ctx,
373        LogEvent::SessionEnd,
374        if resolved.is_ok() {
375            LogResult::Ok
376        } else {
377            LogResult::Err
378        },
379    )?;
380    resolved
381}
382
383fn bookend(ctx: &FetchContext, event: LogEvent, result: LogResult) -> Result<(), FetchError> {
384    ctx.log.append(RowInput {
385        event,
386        result,
387        capability: Capability::Metadata,
388        // No ref: a lookup session is not an answer about a paper, so its
389        // `session_end` must not feed repeat suppression (ADR-0057).
390        ref_: None,
391        source: Some(NCBI),
392        error_code: None,
393        size_bytes: None,
394        license: None,
395        store_path: None,
396        canonical_digest: None,
397    })?;
398    Ok(())
399}
400
401#[cfg(test)]
402#[allow(clippy::expect_used, clippy::unwrap_used, clippy::panic)]
403mod tests {
404    use super::*;
405
406    #[test]
407    fn pubmed_ids_parse_in_their_written_forms_and_bare_digits_do_not() {
408        for (input, want) in [
409            ("pmid:9659853", PubmedId::Pmid(Digits("9659853".into()))),
410            ("PMID: 9659853", PubmedId::Pmid(Digits("9659853".into()))),
411            (
412                "pmcid:PMC3531190",
413                PubmedId::Pmcid(Digits("3531190".into())),
414            ),
415            ("PMC3531190", PubmedId::Pmcid(Digits("3531190".into()))),
416            (
417                "https://pubmed.ncbi.nlm.nih.gov/9659853/",
418                PubmedId::Pmid(Digits("9659853".into())),
419            ),
420            (
421                "https://www.ncbi.nlm.nih.gov/pmc/articles/PMC3531190/",
422                PubmedId::Pmcid(Digits("3531190".into())),
423            ),
424            (
425                "https://pmc.ncbi.nlm.nih.gov/articles/PMC3531190/",
426                PubmedId::Pmcid(Digits("3531190".into())),
427            ),
428        ] {
429            assert_eq!(PubmedId::parse(input), Some(want), "{input}");
430        }
431        for no in [
432            "9659853",
433            "pmid:",
434            "pmid:12a",
435            "10.1176/ajp.155.7.895",
436            "https://example.org/9659853",
437        ] {
438            assert_eq!(PubmedId::parse(no), None, "{no}");
439        }
440        assert_eq!(
441            PubmedId::from_kind("PMCID", "PMC3531190"),
442            Some(PubmedId::Pmcid(Digits("3531190".into())))
443        );
444        assert_eq!(PubmedId::from_kind("ISBN", "x"), None);
445    }
446
447    /// Shapes measured against the live API on 2026-09-29.
448    #[test]
449    fn an_esummary_answer_gives_the_doi_no_doi_or_no_record() {
450        let with_doi = serde_json::json!({"result": {"uids": ["9659853"], "9659853": {
451        "articleids": [
452            {"idtype": "pubmed", "value": "9659853"},
453            {"idtype": "doi", "value": "10.1176/ajp.155.7.895"}
454        ]}}});
455        assert_eq!(
456            classify(&with_doi, "9659853"),
457            Lookup::Doi(Doi::parse("10.1176/ajp.155.7.895").unwrap())
458        );
459        let no_doi = serde_json::json!({"result": {"uids": ["1"], "1": {
460            "articleids": [{"idtype": "pubmed", "value": "1"}]}}});
461        assert_eq!(classify(&no_doi, "1"), Lookup::NoDoi);
462        let missing = serde_json::json!({"result": {"uids": ["99"], "99": {
463            "uid": "99", "error": "cannot get document summary"}}});
464        assert_eq!(classify(&missing, "99"), Lookup::NoRecord);
465        let spaced = serde_json::json!({"result": {"uids": ["2"], "2": {
466            "articleids": [{"idtype": "doi", "value": " 10.1176/ajp.155.7.895 "}]}}});
467        assert!(
468            matches!(classify(&spaced, "2"), Lookup::Doi(_)),
469            "a spaced DOI is a DOI"
470        );
471    }
472}
doiget_core/pubmed.rs

doiget_core/
pubmed.rs
