doiget_core/
metadata_quality.rs

1//! U+FFFD in resolver metadata: detect it, and repair it only from a source
2//! the user enabled (#608).
3//!
4//! Some Crossref records carry U+FFFD REPLACEMENT CHARACTER where the
5//! publisher's deposit lost its umlauts (`Zeitschrift f\u{FFFD}r Physik`,
6//! `N\u{FFFD}herungsmethode`). The entry still compiles, so the damage shows up
7//! only in the rendered bibliography. OpenAlex ingests Crossref and carries
8//! the same loss; Semantic Scholar, measured 2026-09-29 on the two DOIs in
9//! the issue, does not.
10//!
11//! **Repair is guarded.** A candidate replaces a damaged value only when it
12//! matches it with each U+FFFD standing for one or two characters and every
13//! other character equal. So `Zeitschrift f\u{FFFD}r Physik` accepts
14//! `Zeitschrift für Physik` and refuses OpenAlex's `The European Physical
15//! Journal A` (the journal's later name), which leaves the field flagged
16//! rather than swapped for a different fact.
17//!
18//! **No new network by default.** Only sources already enabled for this run
19//! (`DOIGET_ENABLE_S2`, `DOIGET_ENABLE_OPENALEX`) are asked, through the
20//! same rate limiter, allowlists and provenance log as any other call.
21
22use std::collections::BTreeMap;
23
24use serde_json::Value;
25
26use crate::source::{FetchContext, Source};
27use crate::store::Metadata;
28use crate::{CapabilityProfile, Ref};
29
30/// U+FFFD REPLACEMENT CHARACTER.
31pub const REPLACEMENT_CHAR: char = '\u{FFFD}';
32
33/// Metadata fields a U+FFFD is looked for in, by their `docs/STORE.md` name.
34const CHECKED_FIELDS: &[&str] = &["title", "authors", "venue", "publisher", "abstract"];
35
36/// The fields of `m` that carry a U+FFFD: title, authors, venue,
37/// publisher, abstract, in that order.
38#[must_use]
39pub fn replacement_char_fields(m: &Metadata) -> Vec<&'static str> {
40    CHECKED_FIELDS
41        .iter()
42        .copied()
43        .filter(|f| match *f {
44            "title" => has_replacement_char(&m.title),
45            "authors" => m.authors.iter().any(|a| has_replacement_char(a)),
46            "venue" => m.venue.as_deref().is_some_and(has_replacement_char),
47            "publisher" => m.publisher.as_deref().is_some_and(has_replacement_char),
48            "abstract" => m.abstract_.as_deref().is_some_and(has_replacement_char),
49            _ => false,
50        })
51        .collect()
52}
53
54/// Whether `s` carries a U+FFFD.
55#[must_use]
56pub fn has_replacement_char(s: &str) -> bool {
57    s.contains(REPLACEMENT_CHAR)
58}
59
60/// Whether `candidate` is `damaged` with its U+FFFD characters restored:
61/// each U+FFFD in `damaged` stands for one or two characters of `candidate`
62/// (a lost UTF-8 sequence may have been one or two code points), and every
63/// other character is equal. A candidate that itself carries a U+FFFD never
64/// matches.
65///
66/// Both strings come from remote answers, so the match is bounded: longer
67/// than [`RESTORE_MAX_CHARS`] never matches, and a candidate whose length
68/// the U+FFFDs cannot account for is refused before any alignment.
69#[must_use]
70pub fn restores(damaged: &str, candidate: &str) -> bool {
71    if has_replacement_char(candidate) || !has_replacement_char(damaged) {
72        return false;
73    }
74    let c: Vec<char> = candidate.chars().collect();
75    let d_len = damaged.chars().count();
76    let lost = damaged.chars().filter(|&ch| ch == REPLACEMENT_CHAR).count();
77    if d_len > RESTORE_MAX_CHARS || c.len() > RESTORE_MAX_CHARS {
78        return false;
79    }
80    // Each U+FFFD stands for one or two characters.
81    if c.len() < d_len || c.len() > d_len + lost {
82        return false;
83    }
84    // reach[j]: candidate[..j] is matched by the prefix of `damaged` read so far.
85    let mut reach = vec![false; c.len() + 1];
86    let mut next = vec![false; c.len() + 1];
87    reach[0] = true;
88    for dc in damaged.chars() {
89        next.fill(false);
90        for j in (0..=c.len()).filter(|&j| reach[j]) {
91            if dc == REPLACEMENT_CHAR {
92                for step in 1..=2 {
93                    if j + step <= c.len() {
94                        next[j + step] = true;
95                    }
96                }
97            } else if c.get(j) == Some(&dc) {
98                next[j + 1] = true;
99            }
100        }
101        if !next.contains(&true) {
102            return false;
103        }
104        std::mem::swap(&mut reach, &mut next);
105    }
106    reach[c.len()]
107}
108
109/// The longest title or venue [`restores`] will align (#649 review): far
110/// past any real one, short enough that a runaway answer costs nothing.
111pub const RESTORE_MAX_CHARS: usize = 2_000;
112
113/// What [`repair_with`] did.
114#[derive(Debug, Clone, Default, PartialEq, Eq)]
115pub struct QualityReport {
116    /// Field → key of the source whose value replaced the damaged one.
117    pub repaired: BTreeMap<String, String>,
118    /// Fields still carrying a U+FFFD after repair.
119    pub remaining: Vec<&'static str>,
120}
121
122impl QualityReport {
123    /// The machine-readable flags for [`QualityReport::remaining`], e.g.
124    /// `replacement_char:venue`: the `metadata_quality` list of the fetch
125    /// envelopes.
126    #[must_use]
127    pub fn flags(&self) -> Vec<String> {
128        self.remaining
129            .iter()
130            .map(|f| format!("replacement_char:{f}"))
131            .collect()
132    }
133}
134
135/// Repair the U+FFFD fields of `m` that a source can supply (`title` and
136/// `venue`), asking each of `sources` that can serve the DOI at most once,
137/// and record each repair in `[doiget].repaired_fields`.
138///
139/// Authors, publisher and abstract are detected but never repaired: no
140/// enabled source returns them in a shape that can be compared character by
141/// character with Crossref's (`Family, Given`). A source that fails is
142/// skipped with a warning; repair is best-effort and never fails the call.
143pub async fn repair_with(
144    m: &mut Metadata,
145    sources: &[&dyn Source],
146    profile: &CapabilityProfile,
147    ctx: &FetchContext,
148) -> QualityReport {
149    let mut report = QualityReport::default();
150    let repairable: Vec<&'static str> = replacement_char_fields(m)
151        .into_iter()
152        .filter(|f| matches!(*f, "title" | "venue"))
153        .collect();
154    if let (false, Some(doi)) = (repairable.is_empty(), m.doi.clone()) {
155        let ref_ = Ref::Doi(doi);
156        for src in sources {
157            if repairable.iter().all(|f| report.repaired.contains_key(*f)) {
158                break;
159            }
160            if !src.can_serve(profile, &ref_) {
161                continue;
162            }
163            let work = match src.fetch(&ref_, profile, ctx).await {
164                Ok(r) => r.metadata_json,
165                Err(e) => {
166                    tracing::warn!(
167                        source = src.name(),
168                        error = %e,
169                        "could not ask this source to repair a U+FFFD in the metadata"
170                    );
171                    continue;
172                }
173            };
174            let Some(work) = work else { continue };
175            for field in &repairable {
176                if report.repaired.contains_key(*field) {
177                    continue;
178                }
179                let Some(found) = candidate(src.name(), field, &work) else {
180                    continue;
181                };
182                let slot = match *field {
183                    "title" => &mut m.title,
184                    _ => match m.venue.as_mut() {
185                        Some(v) => v,
186                        None => continue,
187                    },
188                };
189                if restores(slot, &found) {
190                    *slot = found;
191                    report
192                        .repaired
193                        .insert((*field).to_string(), src.name().to_string());
194                }
195            }
196        }
197    }
198    report.remaining = replacement_char_fields(m);
199    if let Some(d) = m.doiget.as_mut() {
200        d.repaired_fields.extend(report.repaired.clone());
201    }
202    report
203}
204
205/// Repair `m` from the production Semantic Scholar and OpenAlex sources,
206/// each asked only if enabled in `profile`. See [`repair_with`].
207///
208/// Both sources are compiled in only with the `metadata` feature; without
209/// it this detects and reports, and repairs nothing.
210pub async fn repair(
211    m: &mut Metadata,
212    profile: &CapabilityProfile,
213    ctx: &FetchContext,
214) -> QualityReport {
215    if replacement_char_fields(m).is_empty() {
216        return QualityReport::default();
217    }
218    #[cfg(not(feature = "metadata"))]
219    {
220        let _ = (profile, ctx);
221        QualityReport {
222            remaining: replacement_char_fields(m),
223            ..QualityReport::default()
224        }
225    }
226    #[cfg(feature = "metadata")]
227    {
228        let s2 = crate::sources::s2::S2Source::new(
229            std::env::var("DOIGET_S2_API_KEY")
230                .ok()
231                .filter(|k| !k.is_empty()),
232        );
233        let openalex = crate::sources::openalex::OpenalexSource::new(
234            crate::orchestrator::resolve_contact_email(),
235        );
236        repair_with(m, &[&s2, &openalex], profile, ctx).await
237    }
238}
239
240/// The value `source` reports for `field` in its work record.
241fn candidate(source: &str, field: &str, work: &Value) -> Option<String> {
242    let s = match (source, field) {
243        ("semantic_scholar", "title") => work.get("title")?.as_str()?,
244        ("semantic_scholar", "venue") => work.get("venue")?.as_str()?,
245        ("openalex", "title") => work
246            .get("title")
247            .or_else(|| work.get("display_name"))?
248            .as_str()?,
249        ("openalex", "venue") => work
250            .get("primary_location")?
251            .get("source")?
252            .get("display_name")?
253            .as_str()?,
254        _ => return None,
255    };
256    let s = s.trim();
257    (!s.is_empty()).then(|| s.to_string())
258}
259
260#[cfg(test)]
261#[allow(clippy::expect_used, clippy::unwrap_used)]
262mod tests {
263    use super::*;
264
265    const F: char = REPLACEMENT_CHAR;
266
267    #[test]
268    fn a_candidate_restores_only_the_characters_that_were_lost() {
269        let z = format!("Zeitschrift f{F}r Physik");
270        assert!(restores(&z, "Zeitschrift für Physik"));
271        // The journal's later name is a different fact, not a repair.
272        assert!(!restores(&z, "The European Physical Journal A"));
273        // Other characters must be equal, not merely similar.
274        assert!(!restores(&z, "Zeitschrift für physik"));
275        let n = format!("N{F}herungsmethode zur L{F}sung");
276        assert!(restores(&n, "Näherungsmethode zur Lösung"));
277        // One U+FFFD may stand for two code points (a decomposed umlaut).
278        assert!(restores(&n, "Na\u{308}herungsmethode zur Lösung"));
279        // But never for three, nor for none.
280        assert!(!restores(&n, "Nxyzherungsmethode zur Lösung"));
281        assert!(!restores(&n, "Nherungsmethode zur Lösung"));
282    }
283
284    #[test]
285    fn an_oversized_answer_is_never_aligned() {
286        let long = format!("{}{F}", "a".repeat(RESTORE_MAX_CHARS));
287        assert!(!restores(
288            &long,
289            &format!("{}b", "a".repeat(RESTORE_MAX_CHARS))
290        ));
291        let short = format!("x{F}y");
292        assert!(restores(&short, "xüy"));
293        assert!(!restores(&short, &"x".repeat(50_000)));
294    }
295
296    #[test]
297    fn a_candidate_carrying_the_same_loss_is_not_a_repair() {
298        let z = format!("Zeitschrift f{F}r Physik");
299        assert!(!restores(&z, &z));
300        assert!(!restores(
301            "Zeitschrift für Physik",
302            "Zeitschrift für Physik"
303        ));
304    }
305
306    #[test]
307    fn every_checked_field_is_detected() {
308        let m = Metadata {
309            title: format!("Wechselwirkung neutraler Atome und {F} Bindung"),
310            authors: vec!["London, F.".into(), format!("M{F}ller, A.")],
311            venue: Some(format!("Zeitschrift f{F}r Physik")),
312            publisher: Some("Springer".into()),
313            ..Metadata::default()
314        };
315        assert_eq!(
316            replacement_char_fields(&m),
317            vec!["title", "authors", "venue"]
318        );
319        let report = QualityReport {
320            remaining: vec!["venue"],
321            ..QualityReport::default()
322        };
323        assert_eq!(report.flags(), vec!["replacement_char:venue".to_string()]);
324    }
325
326    /// End to end through a real `Source`: the damaged title is replaced by
327    /// S2's matching one and recorded; the venue S2 does not carry stays
328    /// flagged; a disabled source is never asked.
329    #[cfg(feature = "metadata")]
330    #[tokio::test]
331    async fn repair_takes_a_matching_title_records_it_and_leaves_the_rest_flagged() {
332        use std::sync::Arc;
333
334        use camino::Utf8PathBuf;
335        use wiremock::matchers::{method, path};
336        use wiremock::{Mock, MockServer, ResponseTemplate};
337
338        use crate::http::HttpClient;
339        use crate::provenance::ProvenanceLog;
340        use crate::rate_limiter::RateLimiter;
341        use crate::sources::s2::S2Source;
342        use crate::store::DoigetExtension;
343        use crate::{Doi, RateLimits};
344
345        let server = MockServer::start().await;
346        Mock::given(method("GET"))
347            .and(path("/graph/v1/paper/DOI:10.1007/BF01340294"))
348            .respond_with(ResponseTemplate::new(200).set_body_string(
349                r#"{"paperId": "bf0e", "title": "Näherungsmethode zur Lösung", "venue": ""}"#,
350            ))
351            .expect(1)
352            .mount(&server)
353            .await;
354        let td = tempfile::TempDir::new().expect("tempdir");
355        let log_path = Utf8PathBuf::try_from(td.path().join("t.jsonl")).expect("utf-8");
356        let session_id = "01J0000000000000000000TEST".to_string();
357        let ctx = FetchContext {
358            http: Arc::new(HttpClient::new_for_tests_allow_http(
359                "semantic_scholar",
360                &server.uri(),
361            )),
362            rate_limiter: Arc::new(RateLimiter::new(RateLimits::HARD_CODED)),
363            log: Arc::new(ProvenanceLog::open(log_path, session_id.clone()).expect("log")),
364            session_id,
365            cache_root: None,
366        };
367        let s2 = S2Source::with_base(url::Url::parse(&server.uri()).expect("uri"), None);
368
369        let mut m = Metadata {
370            title: format!("N{F}herungsmethode zur L{F}sung"),
371            venue: Some(format!("Zeitschrift f{F}r Physik")),
372            doi: Some(Doi::parse("10.1007/BF01340294").expect("doi")),
373            doiget: Some(DoigetExtension {
374                fetched_at: chrono::Utc::now(),
375                source: "crossref".into(),
376                license: "unknown".into(),
377                oa_status: None,
378                size_bytes: 0,
379                mcp_call_id: None,
380                tags: Vec::new(),
381                collections: Vec::new(),
382                annotation: None,
383                repaired_fields: BTreeMap::new(),
384                short_venue: None,
385                origin: None,
386            }),
387            ..Metadata::default()
388        };
389
390        // S2 disabled: nothing is asked (`expect(1)` below counts the call).
391        let off = CapabilityProfile::for_tests();
392        let report = repair_with(&mut m.clone(), &[&s2], &off, &ctx).await;
393        assert!(report.repaired.is_empty());
394        assert_eq!(report.remaining, vec!["title", "venue"]);
395
396        let mut on = CapabilityProfile::for_tests();
397        on.metadata.semantic_scholar = true;
398        let report = repair_with(&mut m, &[&s2], &on, &ctx).await;
399        assert_eq!(m.title, "Näherungsmethode zur Lösung");
400        assert_eq!(
401            m.venue.as_deref(),
402            Some(format!("Zeitschrift f{F}r Physik").as_str())
403        );
404        assert_eq!(
405            report.repaired.get("title").map(String::as_str),
406            Some("semantic_scholar")
407        );
408        assert_eq!(report.remaining, vec!["venue"]);
409        assert_eq!(
410            m.doiget.as_ref().map(|d| d.repaired_fields.clone()),
411            Some(report.repaired.clone())
412        );
413    }
414
415    #[test]
416    fn candidates_are_read_from_each_source_shape() {
417        let s2 = serde_json::json!({"paperId": "x", "title": " Näherungsmethode ", "venue": ""});
418        assert_eq!(
419            candidate("semantic_scholar", "title", &s2).as_deref(),
420            Some("Näherungsmethode")
421        );
422        assert_eq!(candidate("semantic_scholar", "venue", &s2), None);
423        let oa = serde_json::json!({
424            "display_name": "Näherungsmethode",
425            "primary_location": {"source": {"display_name": "Zeitschrift für Physik"}}
426        });
427        assert_eq!(
428            candidate("openalex", "title", &oa).as_deref(),
429            Some("Näherungsmethode")
430        );
431        assert_eq!(
432            candidate("openalex", "venue", &oa).as_deref(),
433            Some("Zeitschrift für Physik")
434        );
435    }
436}
doiget_core/metadata_quality.rs

doiget_core/
metadata_quality.rs
