doiget_core/
user_pdf.rs

1//! Add a PDF the user downloaded themselves to the store (#606).
2//!
3//! For a paper with no OA copy, the user downloads it under their own access
4//! and wants the store to know, so no other project hits the same wall. What
5//! this may and may not check is fixed by ADR-0003: **the PDF is an opaque
6//! blob**, so nothing here reads its text, its metadata streams or its page
7//! structure. The checks are the ones a fetch already makes of bytes off the
8//! wire, plus one about the file's *name*:
9//!
10//! - a regular file, not a symlink, starting with `%PDF-`, under
11//!   [`crate::PDF_MAX_BYTES`];
12//! - a file name that is itself a work id -- `BF01397394.pdf`,
13//!   `PhysRev.34.1293.pdf` -- must be the id of the ref it is added under.
14//!   A name like `1928-024 PCPS Hartree - The wave mechanics.pdf` makes no
15//!   claim and is accepted. `force` overrides the mismatch.
16//!
17//! The entry is recorded as what it is: `[doiget] source = "user"`,
18//! `license = "unknown"`, `origin = "user-supplied"` -- never as an open
19//! copy, since `bib`, `paper_pdf_path` and an agent would otherwise read a
20//! licensed download as free to use. The file is copied, not moved, and the
21//! provenance row names the store path only: the original path, which
22//! carries the user's home directory, is not logged.
23
24use camino::{Utf8Path, Utf8PathBuf};
25use chrono::Utc;
26
27use crate::orchestrator::{cite_metadata, resolve_only, write_metadata_and_pdf};
28use crate::source::{FetchContext, FetchError};
29use crate::store::{DoigetExtension, Metadata, Store, StoreError, ORIGIN_USER_SUPPLIED};
30use crate::{CapabilityProfile, Ref};
31
32/// What [`add_user_pdf`] stored.
33#[derive(Debug, Clone)]
34pub struct AddOutcome {
35    /// The entry's safekey.
36    pub safekey: String,
37    /// Where the PDF now lives in the store.
38    pub path: Utf8PathBuf,
39    /// Its size.
40    pub size_bytes: u64,
41    /// The title it was recorded under, for the confirmation line.
42    pub title: String,
43    /// Whether a PDF already in the store was replaced (`force`).
44    pub replaced: bool,
45}
46
47/// Why a file was not added.
48#[derive(Debug, thiserror::Error)]
49pub enum AddError {
50    /// Not a regular file (missing, a directory, a device, ...).
51    #[error("{0} is not a regular file")]
52    NotAFile(Utf8PathBuf),
53    /// A symlink: resolved elsewhere, it is not the file the user named.
54    #[error("{0} is a symbolic link; pass the file it points to")]
55    Symlink(Utf8PathBuf),
56    /// Reading the file failed.
57    #[error("reading {path}: {source}")]
58    Io {
59        /// The file.
60        path: Utf8PathBuf,
61        /// The error.
62        #[source]
63        source: std::io::Error,
64    },
65    /// The first bytes are not `%PDF-` -- the same check a fetch makes.
66    #[error("{0} does not start with %PDF-, so it is not a PDF")]
67    NotAPdf(Utf8PathBuf),
68    /// Larger than a fetch would accept.
69    #[error("{path} is {actual} bytes, over the {cap}-byte cap a fetched PDF is held to")]
70    TooLarge {
71        /// The file.
72        path: Utf8PathBuf,
73        /// Its size.
74        actual: u64,
75        /// [`crate::PDF_MAX_BYTES`].
76        cap: u64,
77    },
78    /// The store already holds a PDF for this ref.
79    #[error("the store already holds a PDF for {ref_} at {path}; pass --force to replace it")]
80    AlreadyStored {
81        /// The ref.
82        ref_: String,
83        /// The stored PDF.
84        path: Utf8PathBuf,
85    },
86    /// The file name is another work's id.
87    #[error(
88        "the file name {stem:?} looks like the id of a different work than {ref_}; \
89         check the download, or pass --force if the name is wrong and the file is right"
90    )]
91    NamesAnotherWork {
92        /// The file name without `.pdf`.
93        stem: String,
94        /// The ref it was being added under.
95        ref_: String,
96    },
97    /// The ref's metadata could not be resolved (and the store had none).
98    #[error("resolving {ref_}: {source}")]
99    Resolve {
100        /// The ref.
101        ref_: String,
102        /// The resolver error.
103        #[source]
104        source: FetchError,
105    },
106    /// The store write failed.
107    #[error("writing the store: {0}")]
108    Store(#[source] FetchError),
109    /// The store has an entry for the ref that cannot be read. Writing over
110    /// it would drop its tags, collections and annotation unseen.
111    #[error("the store entry for {ref_} could not be read ({source}); fix or remove it first")]
112    UnreadableEntry {
113        /// The ref.
114        ref_: String,
115        /// Why the read failed.
116        #[source]
117        source: StoreError,
118    },
119}
120
121/// The file's name without its `.pdf` extension (case-insensitive) and a
122/// browser's duplicate suffix (`name (1).pdf`).
123#[must_use]
124pub fn file_stem(file: &Utf8Path) -> String {
125    let name = file.file_name().unwrap_or("");
126    let stem = name
127        .len()
128        .checked_sub(4)
129        .filter(|&i| name.is_char_boundary(i) && name[i..].eq_ignore_ascii_case(".pdf"))
130        .map_or(name, |i| &name[..i]);
131    let stem = match stem.rfind(" (") {
132        Some(i)
133            if stem.ends_with(')')
134                && stem[i + 2..stem.len() - 1]
135                    .chars()
136                    .all(|c| c.is_ascii_digit()) =>
137        {
138            &stem[..i]
139        }
140        _ => stem,
141    };
142    stem.trim().to_string()
143}
144
145/// Whether `stem` names `ref_`: the DOI suffix (`BF01340294` for
146/// `10.1007/BF01340294`), the whole DOI with `/` as `_` or `-`, or the arXiv
147/// id with or without a version. Case-insensitive.
148#[must_use]
149pub fn stem_names(stem: &str, ref_: &Ref) -> bool {
150    let s = stem.to_lowercase();
151    match ref_ {
152        Ref::Doi(d) => {
153            let doi = d.as_str().to_lowercase();
154            let suffix = doi.split_once('/').map_or(doi.as_str(), |(_, x)| x);
155            s == suffix || s == doi.replace('/', "_") || s == doi.replace('/', "-")
156        }
157        Ref::Arxiv(id) => {
158            let id = id.as_str().to_lowercase().replace('/', "_");
159            let unversioned = match s.rfind('v') {
160                Some(i)
161                    if i > 0
162                        && i + 1 < s.len()
163                        && s[i + 1..].chars().all(|c| c.is_ascii_digit()) =>
164                {
165                    &s[..i]
166                }
167                _ => s.as_str(),
168            };
169            unversioned == id
170                || unversioned == format!("arxiv_{id}")
171                || unversioned == format!("arxiv-{id}")
172        }
173    }
174}
175
176/// Whether `stem` reads as a work id rather than a description: six or more
177/// characters, only letters, digits, `.`, `-` and `_`, and at least one
178/// digit. `BF01397394`, `PhysRev.34.1293` and `rspa.1950.0036` are ids;
179/// `1928-024 PCPS Hartree - The wave mechanics` is not.
180#[must_use]
181pub fn looks_like_an_id(stem: &str) -> bool {
182    stem.len() >= 6
183        && stem
184            .chars()
185            .all(|c| c.is_ascii_alphanumeric() || matches!(c, '.' | '-' | '_'))
186        && stem.chars().any(|c| c.is_ascii_digit())
187}
188
189/// Check `file` the way a fetch checks bytes off the wire, and return the
190/// bytes that passed -- the only ones the store is given. Nothing past the
191/// magic number is interpreted (ADR-0003); the bytes are held, as a fetch
192/// holds a response body, never parsed.
193///
194/// # Errors
195///
196/// [`AddError::NotAFile`], [`AddError::Symlink`], [`AddError::TooLarge`],
197/// [`AddError::NotAPdf`] or [`AddError::Io`].
198pub fn check_file(file: &Utf8Path) -> Result<Vec<u8>, AddError> {
199    use std::io::Read;
200    let io = |source| AddError::Io {
201        path: file.to_path_buf(),
202        source,
203    };
204    let meta = std::fs::symlink_metadata(file).map_err(io)?;
205    if meta.file_type().is_symlink() {
206        return Err(AddError::Symlink(file.to_path_buf()));
207    }
208    if !meta.is_file() {
209        return Err(AddError::NotAFile(file.to_path_buf()));
210    }
211    let too_large = |actual| AddError::TooLarge {
212        path: file.to_path_buf(),
213        actual,
214        cap: crate::PDF_MAX_BYTES,
215    };
216    if meta.len() > crate::PDF_MAX_BYTES {
217        return Err(too_large(meta.len()));
218    }
219    // Everything below reads through ONE handle, and these are the only
220    // bytes that reach the store: a file swapped for a symlink or grown
221    // after the checks above cannot slip past them at copy time.
222    let f = std::fs::File::open(file).map_err(io)?;
223    let opened = f.metadata().map_err(io)?;
224    if !opened.is_file() || !same_file(&meta, &opened) {
225        return Err(AddError::Symlink(file.to_path_buf()));
226    }
227    let mut bytes = Vec::new();
228    f.take(crate::PDF_MAX_BYTES + 1)
229        .read_to_end(&mut bytes)
230        .map_err(io)?;
231    let len = bytes.len() as u64;
232    if len > crate::PDF_MAX_BYTES {
233        return Err(too_large(len));
234    }
235    if !bytes.starts_with(b"%PDF-") {
236        return Err(AddError::NotAPdf(file.to_path_buf()));
237    }
238    Ok(bytes)
239}
240
241/// Whether the path checked and the handle opened are one file.
242#[cfg(unix)]
243fn same_file(a: &std::fs::Metadata, b: &std::fs::Metadata) -> bool {
244    use std::os::unix::fs::MetadataExt;
245    a.dev() == b.dev() && a.ino() == b.ino()
246}
247
248/// Whether the path checked and the handle opened are one file. Stable std
249/// has no file id on Windows; the handle's own `is_file` is the check.
250#[cfg(not(unix))]
251fn same_file(_: &std::fs::Metadata, _: &std::fs::Metadata) -> bool {
252    true
253}
254
255/// Add `file` to the store as `ref_`'s PDF.
256///
257/// Metadata comes from the store when the ref is already there, else from
258/// the same resolver `cite` uses. `force` replaces a stored PDF and
259/// overrides a file name that names another work.
260///
261/// # Errors
262///
263/// Any [`AddError`]; nothing is written unless every check passed.
264pub async fn add_user_pdf(
265    ref_: &Ref,
266    file: &Utf8Path,
267    force: bool,
268    profile: &CapabilityProfile,
269    ctx: &FetchContext,
270    store: &dyn Store,
271    store_root: &Utf8Path,
272) -> Result<AddOutcome, AddError> {
273    let bytes = check_file(file)?;
274    let size = bytes.len() as u64;
275    let stem = file_stem(file);
276    if !force && looks_like_an_id(&stem) && !stem_names(&stem, ref_) {
277        return Err(AddError::NamesAnotherWork {
278            stem,
279            ref_: ref_.as_input_str().to_string(),
280        });
281    }
282    let safekey = ref_.safekey();
283    let pdf_path = store_root.join(format!("{}.pdf", safekey.as_str()));
284    let replaced = pdf_path.exists();
285    if replaced && !force {
286        return Err(AddError::AlreadyStored {
287            ref_: ref_.as_input_str().to_string(),
288            path: pdf_path,
289        });
290    }
291    let stored = crate::store::blocking_section(|| store.read(&safekey)).map_err(|source| {
292        AddError::UnreadableEntry {
293            ref_: ref_.as_input_str().to_string(),
294            source,
295        }
296    })?;
297    let mut m: Metadata = match stored {
298        Some(m) => m,
299        None => {
300            let outcome =
301                resolve_only(ref_, profile, ctx)
302                    .await
303                    .map_err(|source| AddError::Resolve {
304                        ref_: ref_.as_input_str().to_string(),
305                        source,
306                    })?;
307            cite_metadata(ref_, &outcome)
308        }
309    };
310    let prior = m.doiget.take();
311    m.pdf_path = Some(format!("{}.pdf", safekey.as_str()));
312    m.doiget = Some(DoigetExtension {
313        fetched_at: Utc::now(),
314        source: "user".to_string(),
315        license: crate::store::metadata::LICENSE_UNDETERMINED.to_string(),
316        oa_status: prior.as_ref().and_then(|d| d.oa_status.clone()),
317        size_bytes: size,
318        mcp_call_id: None,
319        tags: prior.as_ref().map(|d| d.tags.clone()).unwrap_or_default(),
320        collections: prior
321            .as_ref()
322            .map(|d| d.collections.clone())
323            .unwrap_or_default(),
324        annotation: prior.as_ref().and_then(|d| d.annotation.clone()),
325        repaired_fields: prior
326            .as_ref()
327            .map(|d| d.repaired_fields.clone())
328            .unwrap_or_default(),
329        short_venue: prior.as_ref().and_then(|d| d.short_venue.clone()),
330        origin: Some(ORIGIN_USER_SUPPLIED.to_string()),
331    });
332    let staged = crate::orchestrator::stage_pdf_to_tempfile(&bytes).map_err(AddError::Store)?;
333    let staged_path = Utf8Path::from_path(staged.path()).ok_or_else(|| {
334        AddError::Store(FetchError::SourceSchema {
335            hint: "staging tempfile path is not UTF-8".to_string(),
336        })
337    })?;
338    crate::store::blocking_section(|| {
339        write_metadata_and_pdf(store, &safekey, &m, Some(staged_path), ctx)
340    })
341    .map_err(AddError::Store)?;
342    Ok(AddOutcome {
343        safekey: safekey.as_str().to_string(),
344        path: pdf_path,
345        size_bytes: size,
346        title: m.title,
347        replaced,
348    })
349}
350
351/// The one ref in `candidates` whose id `stem` is, if exactly one (#606
352/// `--from-dir`). Two candidates sharing a DOI suffix is ambiguous and
353/// matches none.
354#[must_use]
355pub fn match_file<'a>(stem: &str, candidates: &'a [Ref]) -> Option<&'a Ref> {
356    let mut hits = candidates.iter().filter(|r| stem_names(stem, r));
357    let first = hits.next()?;
358    hits.next().is_none().then_some(first)
359}
360
361#[cfg(test)]
362#[allow(clippy::expect_used, clippy::unwrap_used)]
363mod tests {
364    use super::*;
365
366    fn r(s: &str) -> Ref {
367        Ref::parse(s).expect("ref")
368    }
369
370    /// The download names from the issue.
371    #[test]
372    fn publisher_download_names_match_their_dois() {
373        for (name, doi) in [
374            ("BF01340294.pdf", "10.1007/BF01340294"),
375            ("PhysRev.34.1293.pdf", "10.1103/PhysRev.34.1293"),
376            ("RevModPhys.23.69.pdf", "10.1103/RevModPhys.23.69"),
377            ("rspa.1950.0036.pdf", "10.1098/rspa.1950.0036"),
378            ("BF01340294 (1).pdf", "10.1007/BF01340294"),
379            ("10.1007_BF01340294.PDF", "10.1007/BF01340294"),
380        ] {
381            let stem = file_stem(Utf8Path::new(name));
382            assert!(stem_names(&stem, &r(doi)), "{name} -> {stem}");
383        }
384        assert!(stem_names("2401.12345v2", &r("arxiv:2401.12345")));
385        assert!(stem_names("cond-mat_0409292", &r("cond-mat/0409292")));
386    }
387
388    #[test]
389    fn a_descriptive_name_makes_no_claim_and_an_id_name_does() {
390        assert!(!looks_like_an_id(&file_stem(Utf8Path::new(
391            "1928-024 PCPS Hartree - The wave mechanics of an atom.pdf"
392        ))));
393        assert!(looks_like_an_id("BF01397394"));
394        assert!(looks_like_an_id("PhysRev.34.1293"));
395        assert!(!looks_like_an_id("thesis"));
396    }
397
398    #[test]
399    fn from_dir_matching_requires_exactly_one_candidate() {
400        let c = vec![r("10.1007/BF01340294"), r("10.1103/PhysRev.34.1293")];
401        assert_eq!(
402            match_file("bf01340294", &c).map(Ref::as_input_str),
403            Some("10.1007/BF01340294")
404        );
405        assert!(match_file("unrelated-2020", &c).is_none());
406        let dup = vec![r("10.1007/X123456"), r("10.9999/X123456")];
407        assert!(match_file("X123456", &dup).is_none());
408    }
409
410    #[test]
411    fn the_file_checks_are_the_ones_a_fetch_makes() {
412        let td = tempfile::TempDir::new().expect("tempdir");
413        let dir = Utf8Path::from_path(td.path()).expect("utf-8");
414        let pdf = dir.join("ok.pdf");
415        std::fs::write(&pdf, b"%PDF-1.4\n...").expect("write");
416        assert_eq!(check_file(&pdf).expect("ok").len(), 12);
417        let html = dir.join("login.pdf");
418        std::fs::write(&html, b"<!doctype html>").expect("write");
419        assert!(matches!(check_file(&html), Err(AddError::NotAPdf(_))));
420        assert!(matches!(check_file(dir), Err(AddError::NotAFile(_))));
421        assert!(matches!(
422            check_file(&dir.join("absent.pdf")),
423            Err(AddError::Io { .. })
424        ));
425        let big = dir.join("big.pdf");
426        let f = std::fs::File::create(&big).expect("create");
427        f.set_len(crate::PDF_MAX_BYTES + 1).expect("sparse");
428        assert!(matches!(check_file(&big), Err(AddError::TooLarge { .. })));
429        #[cfg(unix)]
430        {
431            let link = dir.join("link.pdf");
432            std::os::unix::fs::symlink(&pdf, &link).expect("symlink");
433            assert!(matches!(check_file(&link), Err(AddError::Symlink(_))));
434        }
435    }
436}
doiget_core/user_pdf.rs

doiget_core/
user_pdf.rs
