doiget_core/
base_override.rs

1//! The `DOIGET_*_BASE` test / proxy overrides, as one table (#587).
2//!
3//! Every fetch source honours a `DOIGET_<SOURCE>_BASE` that points it at
4//! another origin: a wiremock server in tests, an institutional proxy in the
5//! field. The HTTP client has to agree with the sources about that, and it
6//! used to do so through two hand-maintained copies (the CLI's
7//! `build_http_client` and the MCP server's `build_http_client_for_fetch`),
8//! each with a selection condition and a registration list. The lists knew
9//! six to eleven of the fifteen overridable keys, so setting any one base put
10//! the process in test mode and silently dropped the rest from the client --
11//! `datacite`, `hal`, `openaire`, `core` and `europe-pmc` could not be mocked
12//! at all.
13//!
14//! [`BASE_OVERRIDES`] is now the only list, and [`test_client_from_env`] the
15//! only builder that reads it.
16//!
17//! ## Which variables switch to the test client
18//!
19//! Only the rows marked [`BaseOverride::selects_test_client`] -- the same
20//! seven the condition always tested. The others are honoured *within* test
21//! mode but never cause it, and that asymmetry is deliberate: the production
22//! client is `https_only` and applies its allowlists to redirects, so
23//! `DOIGET_APS_BASE=https://proxy.example.edu` alone works today on the
24//! production client, with every other source intact. Making it select the
25//! test client would rebuild the process from the overrides alone and drop
26//! Crossref, which is the regression the issue's option 2 would have
27//! shipped.
28//!
29//! Test mode stays **exclusive**: a source with no override is absent from
30//! the test client, so a test that forgot to mock one fails immediately and
31//! offline with `UnknownSource` instead of reaching the real API. That
32//! isolation is the property the issue's first-proposed additive fix would
33//! have lost.
34
35use crate::http::HttpClient;
36
37/// One overridable source.
38#[derive(Debug, Clone, Copy, PartialEq, Eq)]
39pub struct BaseOverride {
40    /// The HTTP client's source key, i.e. the key the source passes to
41    /// `HttpClient::fetch_*`.
42    pub source: &'static str,
43    /// The environment variable naming the replacement base URL.
44    pub env: &'static str,
45    /// Whether setting this variable switches the whole process to the
46    /// allow-http test client. See the module docs for why only some do.
47    pub selects_test_client: bool,
48}
49
50const fn row(source: &'static str, env: &'static str, selects: bool) -> BaseOverride {
51    BaseOverride {
52        source,
53        env,
54        selects_test_client: selects,
55    }
56}
57
58/// Every `DOIGET_*_BASE` a fetch source reads. Rows sharing a `source` are
59/// alternatives, the first one set winning: `DOIGET_ARXIV_SRC_BASE` is the
60/// source-bundle endpoint's override and stands in for `DOIGET_ARXIV_BASE`
61/// only when that is unset.
62pub const BASE_OVERRIDES: &[BaseOverride] = &[
63    // Tier 1 and the always-on metadata / full-text keys.
64    row("arxiv", "DOIGET_ARXIV_BASE", true),
65    row("arxiv", "DOIGET_ARXIV_SRC_BASE", true),
66    row("crossref", "DOIGET_CROSSREF_BASE", true),
67    row("unpaywall", "DOIGET_UNPAYWALL_BASE", true),
68    row("oa-publisher", "DOIGET_OA_PUBLISHER_BASE", true),
69    row("openalex", "DOIGET_OPENALEX_BASE", true),
70    row("ar5iv", "DOIGET_AR5IV_BASE", true),
71    // A published DOI's bioRxiv / medRxiv preprint (#640).
72    row("biorxiv", "DOIGET_BIORXIV_BASE", false),
73    row("inspire", "DOIGET_INSPIRE_BASE", false),
74    row("ads", "DOIGET_ADS_BASE", false),
75    // PubMed id -> DOI (#500).
76    row("ncbi", "DOIGET_NCBI_BASE", true),
77    // `cite <github URL>` (#614).
78    row("github", "DOIGET_GITHUB_API_BASE", true),
79    row("github-raw", "DOIGET_GITHUB_RAW_BASE", true),
80    // Tier 2 optional chain (`metadata`).
81    row("datacite", "DOIGET_DATACITE_BASE", false),
82    row("europe-pmc", "DOIGET_EUROPE_PMC_BASE", false),
83    row("openaire", "DOIGET_OPENAIRE_BASE", false),
84    row("hal", "DOIGET_HAL_BASE", false),
85    row("core", "DOIGET_CORE_BASE", false),
86    // Tier 3 (`tdm-*`). Registering a key whose source is not compiled in
87    // is harmless: nothing asks for it.
88    row("tdm-aps", "DOIGET_APS_BASE", false),
89    row("tdm-elsevier", "DOIGET_ELSEVIER_BASE", false),
90    row("tdm-springer", "DOIGET_SPRINGER_BASE", false),
91    row("tdm-ieee", "DOIGET_IEEE_BASE", false),
92];
93
94/// Why the override environment could not become a client.
95#[derive(Debug, thiserror::Error)]
96pub enum BaseOverrideError {
97    /// A `DOIGET_*_BASE` is set but is not an absolute URL with a host.
98    #[error("{env} is not a URL with a host: {value:?}")]
99    NotAUrl {
100        /// The variable.
101        env: &'static str,
102        /// What it was set to.
103        value: String,
104    },
105}
106
107/// The allow-http test client for the overrides set in the environment, or
108/// `None` when no [`BaseOverride::selects_test_client`] variable is set and
109/// the caller should build its production client.
110///
111/// In test mode every row that is set is registered -- selecting or not --
112/// and nothing else is: see the module docs on exclusivity.
113///
114/// # Errors
115///
116/// [`BaseOverrideError::NotAUrl`] for a set variable that does not parse, in
117/// test mode. A malformed non-selecting variable outside test mode is the
118/// source's to report, as before.
119pub fn test_client_from_env() -> Result<Option<HttpClient>, BaseOverrideError> {
120    test_client_from(|env| std::env::var(env).ok())
121}
122
123/// [`test_client_from_env`] over an arbitrary variable lookup, for tests.
124///
125/// # Errors
126///
127/// As [`test_client_from_env`].
128pub fn test_client_from(
129    lookup: impl Fn(&str) -> Option<String>,
130) -> Result<Option<HttpClient>, BaseOverrideError> {
131    match test_entries(lookup)? {
132        Some(entries) => {
133            let borrowed: Vec<(&str, &str)> =
134                entries.iter().map(|(s, h)| (*s, h.as_str())).collect();
135            Ok(Some(HttpClient::new_for_tests_allow_http_multi(&borrowed)))
136        }
137        None => Ok(None),
138    }
139}
140
141/// The `(source key, host)` pairs the test client registers, or `None` for
142/// production. Split out so the selection logic is testable without
143/// building a client.
144fn test_entries(
145    lookup: impl Fn(&str) -> Option<String>,
146) -> Result<Option<Vec<(&'static str, String)>>, BaseOverrideError> {
147    let set: Vec<(BaseOverride, String)> = BASE_OVERRIDES
148        .iter()
149        .filter_map(|o| lookup(o.env).map(|v| (*o, v)))
150        .collect();
151    if !set.iter().any(|(o, _)| o.selects_test_client) {
152        return Ok(None);
153    }
154    let mut entries: Vec<(&'static str, String)> = Vec::new();
155    for (o, value) in set {
156        if entries.iter().any(|(s, _)| *s == o.source) {
157            continue;
158        }
159        let host = url::Url::parse(&value)
160            .ok()
161            .and_then(|u| u.host_str().map(str::to_string))
162            .ok_or_else(|| BaseOverrideError::NotAUrl {
163                env: o.env,
164                value: value.clone(),
165            })?;
166        entries.push((o.source, host));
167    }
168    Ok(Some(entries))
169}
170
171#[cfg(test)]
172#[allow(clippy::expect_used, clippy::unwrap_used)]
173mod tests {
174    use super::*;
175
176    fn env<'a>(pairs: &'a [(&'a str, &'a str)]) -> impl Fn(&str) -> Option<String> + 'a {
177        move |k| {
178            pairs
179                .iter()
180                .find(|(n, _)| *n == k)
181                .map(|(_, v)| (*v).to_string())
182        }
183    }
184
185    #[test]
186    fn nothing_set_is_production() {
187        assert!(test_entries(env(&[])).unwrap().is_none());
188    }
189
190    /// The institutional-proxy case: a TDM or Tier-2 base alone keeps the
191    /// production client, so every other source keeps its allowlist.
192    #[test]
193    fn a_non_selecting_base_alone_stays_on_the_production_client() {
194        for o in BASE_OVERRIDES.iter().filter(|o| !o.selects_test_client) {
195            let pairs = [(o.env, "https://proxy.example.edu")];
196            assert!(test_entries(env(&pairs)).unwrap().is_none(), "{}", o.env);
197        }
198    }
199
200    /// #587: in test mode, every overridden key is registered -- the Tier-2
201    /// and Tier-3 keys used to be dropped.
202    #[test]
203    fn test_mode_registers_every_overridden_source_and_nothing_else() {
204        let pairs = [
205            ("DOIGET_CROSSREF_BASE", "http://127.0.0.1:9001"),
206            ("DOIGET_DATACITE_BASE", "http://127.0.0.1:9002"),
207            ("DOIGET_EUROPE_PMC_BASE", "http://127.0.0.1:9003"),
208            ("DOIGET_APS_BASE", "http://127.0.0.1:9004"),
209        ];
210        let entries = test_entries(env(&pairs)).unwrap().expect("test mode");
211        let keys: Vec<&str> = entries.iter().map(|(s, _)| *s).collect();
212        assert_eq!(keys, vec!["crossref", "datacite", "europe-pmc", "tdm-aps"]);
213        assert!(entries.iter().all(|(_, h)| h == "127.0.0.1"));
214    }
215
216    #[test]
217    fn the_source_bundle_base_stands_in_for_arxiv_only_when_it_is_unset() {
218        let src_only = [("DOIGET_ARXIV_SRC_BASE", "http://src.test:1")];
219        let entries = test_entries(env(&src_only)).unwrap().expect("test mode");
220        assert_eq!(entries, vec![("arxiv", "src.test".to_string())]);
221        let both = [
222            ("DOIGET_ARXIV_BASE", "http://api.test:1"),
223            ("DOIGET_ARXIV_SRC_BASE", "http://src.test:1"),
224        ];
225        let entries = test_entries(env(&both)).unwrap().expect("test mode");
226        assert_eq!(entries, vec![("arxiv", "api.test".to_string())]);
227    }
228
229    #[test]
230    fn a_malformed_base_in_test_mode_names_its_variable() {
231        let pairs = [
232            ("DOIGET_CROSSREF_BASE", "http://127.0.0.1:1"),
233            ("DOIGET_HAL_BASE", "not a url"),
234        ];
235        let err = test_entries(env(&pairs)).unwrap_err();
236        assert!(
237            err.to_string().starts_with("DOIGET_HAL_BASE is not a URL"),
238            "{err}"
239        );
240    }
241
242    /// The table is only worth having if it is complete: every
243    /// `DOIGET_*_BASE` a fetch source reads must have a row, or setting it
244    /// in test mode drops its key again. `DOIGET_GITHUB_BASE` is
245    /// `doiget version`'s own client, not the fetch client, and lives in
246    /// doiget-cli.
247    #[test]
248    fn every_base_override_read_by_a_source_has_a_row() {
249        let sources = [
250            include_str!("orchestrator.rs"),
251            include_str!("paper_tex_source.rs"),
252            include_str!("paper_text.rs"),
253            include_str!("discovery.rs"),
254            include_str!("citation_graph.rs"),
255        ]
256        .concat();
257        let mut read: Vec<&str> = sources
258            .match_indices("\"DOIGET_")
259            .filter_map(|(i, _)| {
260                let rest = &sources[i + 1..];
261                let name = &rest[..rest.find('"')?];
262                name.ends_with("_BASE").then_some(name)
263            })
264            .collect();
265        read.sort_unstable();
266        read.dedup();
267        assert!(read.len() >= 10, "scan found too few variables: {read:?}");
268        let missing: Vec<&str> = read
269            .into_iter()
270            .filter(|v| !BASE_OVERRIDES.iter().any(|o| o.env == *v))
271            .collect();
272        assert!(
273            missing.is_empty(),
274            "DOIGET_*_BASE without a BASE_OVERRIDES row: {missing:?}"
275        );
276    }
277}
doiget_core/base_override.rs

doiget_core/
base_override.rs
