doiget_core/
source_catalog.rs

1//! Every source doiget knows, whether or not this binary was built with it,
2//! and whether it can serve a given ref right now (#605).
3//!
4//! `fetch --dry-run` lists `candidate_hosts`, and its own help says that is
5//! the static allowlist, "not a prediction". Before deciding whether to wait
6//! for doiget or download a paper by hand, the question is different: can
7//! **any** configured source deliver this DOI, and if not, why not -- not
8//! built, not enabled, no credentials, or the publisher is not one it covers.
9//! [`CATALOG`] is the list that answers it, and it exists in every build:
10//! a default binary has no Tier-3 code (ADR-0002), but it can still say that
11//! `tdm-aps` covers `10.1103` and needs `--features tdm-aps`.
12//!
13//! This is a statement about **reach**, never about outcome: a source that is
14//! `Ready` for a DOI may still find nothing. The coverage report says so.
15
16use crate::{CapabilityProfile, Ref};
17
18/// What a source can contribute to a fetch.
19#[derive(Debug, Clone, Copy, PartialEq, Eq)]
20pub enum Role {
21    /// Bibliographic metadata only; never a PDF.
22    Metadata,
23    /// Where an OA copy lives -- a URL the `oa-publisher` leg then fetches.
24    OaLocation,
25    /// The PDF itself.
26    Content,
27}
28
29impl Role {
30    /// Wire token.
31    #[must_use]
32    pub const fn as_str(self) -> &'static str {
33        match self {
34            Self::Metadata => "metadata",
35            Self::OaLocation => "oa_location",
36            Self::Content => "content",
37        }
38    }
39}
40
41/// Which refs a source is able to answer for.
42#[derive(Debug, Clone, Copy, PartialEq, Eq)]
43pub enum Covers {
44    /// Any DOI.
45    AnyDoi,
46    /// arXiv ids (and DOIs through the preprint fallback).
47    Arxiv,
48    /// DOIs registered with DataCite (Zenodo, figshare, Dryad, OSF, ...).
49    DataCiteDois,
50    /// DOIs under these registrant prefixes only (ADR-0041).
51    Prefixes(&'static [&'static str]),
52}
53
54/// One catalog row.
55#[derive(Debug, Clone, Copy, PartialEq, Eq)]
56pub struct SourceInfo {
57    /// Source key, as in the attempt trace.
58    pub name: &'static str,
59    /// Tier 1 (always on), 2 (opt-in, no key), 3 (TDM agreement + key).
60    pub tier: u8,
61    /// What it contributes.
62    pub role: Role,
63    /// Which refs it answers for.
64    pub covers: Covers,
65    /// The publisher a prefix-scoped source belongs to.
66    pub publisher: Option<&'static str>,
67    /// Cargo feature the source is compiled under, if any.
68    pub feature: Option<&'static str>,
69    /// Environment the source needs at run time (enable flag, key, agreement).
70    pub enable: &'static [&'static str],
71    /// Whether this binary contains the source.
72    pub compiled: bool,
73}
74
75const fn t1(name: &'static str, role: Role, covers: Covers) -> SourceInfo {
76    SourceInfo {
77        name,
78        tier: 1,
79        role,
80        covers,
81        publisher: None,
82        feature: None,
83        enable: &[],
84        compiled: true,
85    }
86}
87
88const fn t2(
89    name: &'static str,
90    role: Role,
91    covers: Covers,
92    enable: &'static [&'static str],
93) -> SourceInfo {
94    SourceInfo {
95        name,
96        tier: 2,
97        role,
98        covers,
99        publisher: None,
100        feature: Some("metadata"),
101        enable,
102        compiled: cfg!(feature = "metadata"),
103    }
104}
105
106/// Every source, in the order a fetch consults them.
107pub const CATALOG: &[SourceInfo] = &[
108    t1("crossref", Role::Metadata, Covers::AnyDoi),
109    t1("unpaywall", Role::OaLocation, Covers::AnyDoi),
110    t1("oa-publisher", Role::Content, Covers::AnyDoi),
111    t1("arxiv", Role::Content, Covers::Arxiv),
112    t2(
113        "datacite",
114        Role::Metadata,
115        Covers::DataCiteDois,
116        &["DOIGET_ENABLE_DATACITE"],
117    ),
118    t2(
119        "europe-pmc",
120        Role::OaLocation,
121        Covers::AnyDoi,
122        &["DOIGET_ENABLE_EUROPE_PMC"],
123    ),
124    t2(
125        "openaire",
126        Role::OaLocation,
127        Covers::AnyDoi,
128        &["DOIGET_ENABLE_OPENAIRE"],
129    ),
130    t2(
131        "hal",
132        Role::OaLocation,
133        Covers::AnyDoi,
134        &["DOIGET_ENABLE_HAL"],
135    ),
136    t2(
137        "core",
138        Role::OaLocation,
139        Covers::AnyDoi,
140        &["DOIGET_ENABLE_CORE", "DOIGET_CORE_API_KEY"],
141    ),
142    t2(
143        "openalex",
144        Role::OaLocation,
145        Covers::AnyDoi,
146        &["DOIGET_ENABLE_OPENALEX"],
147    ),
148    t2(
149        "semantic_scholar",
150        Role::Metadata,
151        Covers::AnyDoi,
152        &["DOIGET_ENABLE_S2"],
153    ),
154    t2(
155        "ads",
156        Role::OaLocation,
157        Covers::AnyDoi,
158        &["DOIGET_ADS_TOKEN"],
159    ),
160    t2(
161        "inspire",
162        Role::OaLocation,
163        Covers::AnyDoi,
164        &["DOIGET_ENABLE_INSPIRE"],
165    ),
166    t2(
167        "biorxiv",
168        Role::OaLocation,
169        Covers::AnyDoi,
170        &["DOIGET_ENABLE_BIORXIV"],
171    ),
172    t2(
173        "doaj",
174        Role::Metadata,
175        Covers::AnyDoi,
176        &["DOIGET_ENABLE_DOAJ"],
177    ),
178    SourceInfo {
179        name: "tdm-aps",
180        tier: 3,
181        role: Role::Content,
182        covers: Covers::Prefixes(&["10.1103"]),
183        publisher: Some("American Physical Society (APS)"),
184        feature: Some("tdm-aps"),
185        enable: &["DOIGET_KEY_APS", "DOIGET_AGREE_TDM_APS"],
186        compiled: cfg!(feature = "tdm-aps"),
187    },
188    SourceInfo {
189        name: "tdm-elsevier",
190        tier: 3,
191        role: Role::Content,
192        covers: Covers::Prefixes(&["10.1016", "10.1006", "10.1053"]),
193        publisher: Some("Elsevier BV"),
194        feature: Some("tdm-elsevier"),
195        enable: &["DOIGET_KEY_ELSEVIER", "DOIGET_AGREE_TDM_ELSEVIER"],
196        compiled: cfg!(feature = "tdm-elsevier"),
197    },
198    SourceInfo {
199        name: "tdm-springer",
200        tier: 3,
201        role: Role::Content,
202        covers: Covers::Prefixes(&["10.1007", "10.1038", "10.1057", "10.1140"]),
203        publisher: Some("Springer Nature"),
204        feature: Some("tdm-springer"),
205        enable: &["DOIGET_KEY_SPRINGER", "DOIGET_AGREE_TDM_SPRINGER"],
206        compiled: cfg!(feature = "tdm-springer"),
207    },
208    SourceInfo {
209        name: "tdm-ieee",
210        tier: 3,
211        role: Role::Content,
212        covers: Covers::Prefixes(&["10.1109", "10.23919"]),
213        publisher: Some("IEEE"),
214        feature: Some("tdm-ieee"),
215        enable: &["DOIGET_KEY_IEEE", "DOIGET_AGREE_TDM_IEEE"],
216        compiled: cfg!(feature = "tdm-ieee"),
217    },
218];
219
220/// Whether a source can be asked about a ref right now.
221#[derive(Debug, Clone, PartialEq, Eq)]
222pub enum Availability {
223    /// Compiled, enabled, and covers the ref. Says nothing about whether it
224    /// will find anything.
225    Ready,
226    /// This binary was built without it.
227    NotBuilt {
228        /// The Cargo feature to build with.
229        feature: &'static str,
230    },
231    /// Built, but its environment is not set.
232    NotEnabled {
233        /// What to set.
234        enable: &'static [&'static str],
235    },
236    /// Built and enabled, but the ref is outside what it covers.
237    NotCovered,
238}
239
240impl Availability {
241    /// Wire token.
242    #[must_use]
243    pub const fn as_str(&self) -> &'static str {
244        match self {
245            Self::Ready => "ready",
246            Self::NotBuilt { .. } => "not_built",
247            Self::NotEnabled { .. } => "not_enabled",
248            Self::NotCovered => "not_covered",
249        }
250    }
251
252    /// One line a person can act on.
253    #[must_use]
254    pub fn remedy(&self) -> Option<String> {
255        match self {
256            Self::Ready | Self::NotCovered => None,
257            Self::NotBuilt { feature } => Some(format!("build with --features {feature}")),
258            Self::NotEnabled { enable } => Some(format!("set {}", enable.join(" and "))),
259        }
260    }
261}
262
263/// Whether `info` covers `ref_` by its own scope, ignoring build and config.
264/// A DataCite-only source is reported as covering every DOI: which agency
265/// registered a DOI is only known by asking.
266#[must_use]
267pub fn covers(info: &SourceInfo, ref_: &Ref) -> bool {
268    match (info.covers, ref_) {
269        // A DOI reaches arXiv only through the preprint fallback (#325), and
270        // only when Unpaywall named the arXiv copy -- which `coverage`
271        // already reports as the open copy. On its own it does not cover one.
272        (Covers::Arxiv, Ref::Arxiv(_)) => true,
273        (Covers::Arxiv, Ref::Doi(_)) | (_, Ref::Arxiv(_)) => false,
274        (Covers::AnyDoi | Covers::DataCiteDois, Ref::Doi(_)) => true,
275        (Covers::Prefixes(p), Ref::Doi(d)) => {
276            let prefix = d.as_str().split('/').next().unwrap_or("");
277            p.contains(&prefix)
278        }
279    }
280}
281
282/// Build and configuration state of `info`, with no ref to be in scope for:
283/// [`Availability::NotBuilt`], [`Availability::NotEnabled`] or
284/// [`Availability::Ready`] -- never `NotCovered`.
285#[must_use]
286pub fn configured(info: &SourceInfo, profile: &CapabilityProfile) -> Availability {
287    if !info.compiled {
288        Availability::NotBuilt {
289            feature: info.feature.unwrap_or("?"),
290        }
291    } else if !enabled(info.name, profile) {
292        Availability::NotEnabled {
293            enable: info.enable,
294        }
295    } else {
296        Availability::Ready
297    }
298}
299
300/// [`Availability`] of `info` for `ref_` under `profile`.
301#[must_use]
302pub fn availability(info: &SourceInfo, profile: &CapabilityProfile, ref_: &Ref) -> Availability {
303    match configured(info, profile) {
304        Availability::Ready if !covers(info, ref_) => Availability::NotCovered,
305        other => other,
306    }
307}
308
309fn enabled(name: &str, p: &CapabilityProfile) -> bool {
310    let m = &p.metadata;
311    match name {
312        "datacite" => m.datacite,
313        "europe-pmc" => m.europe_pmc,
314        "openaire" => m.openaire,
315        "hal" => m.hal,
316        "core" => m.core,
317        "openalex" => m.openalex,
318        "semantic_scholar" => m.semantic_scholar,
319        "doaj" => m.doaj,
320        "biorxiv" => m.biorxiv,
321        "inspire" => m.inspire,
322        "ads" => m.ads,
323        "tdm-aps" => p.tdm_aps.is_some(),
324        "tdm-elsevier" => p.tdm_elsevier.is_some(),
325        "tdm-springer" => p.tdm_springer.is_some(),
326        "tdm-ieee" => p.tdm_ieee.is_some(),
327        _ => true,
328    }
329}
330
331/// Catalog rows relevant to `publisher`: a registrant prefix (`10.1103`) or a
332/// case-insensitive fragment of a publisher's name (`aps`, `springer`).
333/// Sources that cover any DOI are always relevant.
334#[must_use]
335pub fn for_publisher(publisher: &str) -> Vec<&'static SourceInfo> {
336    let q = publisher.trim().to_lowercase();
337    CATALOG
338        .iter()
339        .filter(|s| match s.covers {
340            Covers::Prefixes(p) => {
341                p.iter().any(|x| q == *x || q.starts_with(&format!("{x}/")))
342                    || s.publisher.is_some_and(|n| n.to_lowercase().contains(&q))
343            }
344            Covers::Arxiv => q == "arxiv" || q == "10.48550",
345            _ => true,
346        })
347        .collect()
348}
349
350#[cfg(test)]
351#[allow(clippy::expect_used, clippy::unwrap_used)]
352mod tests {
353    use super::*;
354
355    fn doi(s: &str) -> Ref {
356        Ref::parse(s).expect("ref")
357    }
358
359    #[test]
360    fn a_default_profile_names_the_switch_for_each_source_it_will_not_ask() {
361        let p = CapabilityProfile::for_tests();
362        let r = doi("10.1103/PhysRevB.48.10345");
363        let by = |n: &str| CATALOG.iter().find(|s| s.name == n).expect("row");
364        assert_eq!(availability(by("crossref"), &p, &r), Availability::Ready);
365        let aps = availability(by("tdm-aps"), &p, &r);
366        if cfg!(feature = "tdm-aps") {
367            assert!(matches!(aps, Availability::NotEnabled { .. }));
368        } else {
369            assert_eq!(
370                aps.remedy().as_deref(),
371                Some("build with --features tdm-aps")
372            );
373        }
374        if cfg!(feature = "metadata") {
375            assert_eq!(
376                availability(by("hal"), &p, &r).remedy().as_deref(),
377                Some("set DOIGET_ENABLE_HAL")
378            );
379        }
380    }
381
382    #[test]
383    fn a_publisher_scoped_source_covers_only_its_prefixes() {
384        let aps = CATALOG.iter().find(|s| s.name == "tdm-aps").unwrap();
385        assert!(covers(aps, &doi("10.1103/PhysRevB.48.10345")));
386        assert!(!covers(aps, &doi("10.1007/BF01340294")));
387        assert!(!covers(aps, &doi("arxiv:2401.00001")));
388    }
389
390    #[test]
391    fn a_publisher_query_matches_prefix_or_name_and_keeps_general_sources() {
392        let names = |q: &str| -> Vec<&str> { for_publisher(q).iter().map(|s| s.name).collect() };
393        assert!(names("10.1103").contains(&"tdm-aps"));
394        assert!(!names("10.1103").contains(&"tdm-springer"));
395        assert!(names("Springer").contains(&"tdm-springer"));
396        assert!(names("springer").contains(&"crossref"));
397        assert!(!names("springer").contains(&"arxiv"));
398    }
399
400    /// The catalog repeats the TDM prefixes so a build without the source
401    /// can still name them; it must never disagree with the source itself.
402    #[test]
403    fn the_catalog_prefixes_are_the_sources_own() {
404        #[allow(unused_mut)]
405        let mut pairs: Vec<(&str, &[&str])> = Vec::new();
406        #[cfg(feature = "tdm-aps")]
407        pairs.push(("tdm-aps", crate::sources::tdm_aps::PUBLISHER_PREFIXES));
408        #[cfg(feature = "tdm-elsevier")]
409        pairs.push((
410            "tdm-elsevier",
411            crate::sources::tdm_elsevier::PUBLISHER_PREFIXES,
412        ));
413        #[cfg(feature = "tdm-springer")]
414        pairs.push((
415            "tdm-springer",
416            crate::sources::tdm_springer::PUBLISHER_PREFIXES,
417        ));
418        #[cfg(feature = "tdm-ieee")]
419        pairs.push(("tdm-ieee", crate::sources::tdm_ieee::PUBLISHER_PREFIXES));
420        for (name, own) in pairs {
421            let row = CATALOG.iter().find(|s| s.name == name).expect("row");
422            assert_eq!(row.covers, Covers::Prefixes(own), "{name}");
423        }
424    }
425
426    #[test]
427    fn arxiv_covers_arxiv_ids_and_not_dois() {
428        let arxiv = CATALOG.iter().find(|s| s.name == "arxiv").unwrap();
429        assert!(covers(arxiv, &doi("arXiv:cond-mat/0409292")));
430        assert!(!covers(arxiv, &doi("10.1038/nphys1170")));
431    }
432
433    /// Every module under `src/sources/` has a catalog row, so a new source
434    /// cannot ship invisible to `doiget sources` (DOAJ once did).
435    #[test]
436    fn every_source_module_has_a_catalog_row() {
437        let dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/sources");
438        for entry in std::fs::read_dir(dir).expect("sources dir") {
439            let file = entry.expect("entry").file_name();
440            let stem = file.to_str().unwrap().trim_end_matches(".rs");
441            let name = match stem {
442                "mod" => continue,
443                "core_oa" => "core",
444                "europepmc" => "europe-pmc",
445                "s2" => "semantic_scholar",
446                other => &other.replace('_', "-"),
447            };
448            assert!(
449                CATALOG.iter().any(|s| s.name == name),
450                "src/sources/{stem}.rs has no CATALOG row named {name:?}"
451            );
452        }
453    }
454}
doiget_core/source_catalog.rs

doiget_core/
source_catalog.rs
