1""".. include:: ../README.md"""
  2
  3from __future__ import annotations
  4
  5import logging
  6import time
  7import itertools
  8import requests
  9
 10from importlib.metadata import PackageNotFoundError, version
 11from urllib.parse import urlencode
 12from datetime import datetime, timedelta, timezone
 13from calendar import timegm
 14
 15from enum import Enum
 16from typing import Generator, Iterator
 17
 18from . import _feed
 19from ._feed import ParsedFeed
 20
 21
 22logger = logging.getLogger(__name__)
 23
 24try:
 25    __version__ = version("arxiv")
 26except PackageNotFoundError:
 27    __version__ = "0.0.0+unknown"
 28
 29_USER_AGENT = f"arxiv.py/{__version__}"
 30
 31_DEFAULT_TIME = datetime.min
 32
 33
 34class Result:
 35    """
 36    An entry in an arXiv query results feed.
 37
 38    See [the arXiv API User's Manual: Details of Atom Results
 39    Returned](https://arxiv.org/help/api/user-manual#_details_of_atom_results_returned).
 40    """
 41
 42    entry_id: str
 43    """A url of the form `https://arxiv.org/abs/{id}`."""
 44    updated: datetime
 45    """When the result was last updated."""
 46    published: datetime
 47    """When the result was originally published."""
 48    title: str
 49    """The title of the result."""
 50    authors: list[Result.Author]
 51    """The result's authors, including any `<arxiv:affiliation>` data."""
 52    summary: str
 53    """The result abstract."""
 54    comment: str | None
 55    """The authors' comment if present."""
 56    journal_ref: str | None
 57    """A journal reference if present."""
 58    doi: str | None
 59    """A URL for the resolved DOI to an external resource if present."""
 60    primary_category: str
 61    """
 62    The result's primary arXiv category. See [arXiv: Category
 63    Taxonomy](https://arxiv.org/category_taxonomy).
 64    """
 65    categories: list[str]
 66    """
 67    All of the result's categories. See [arXiv: Category
 68    Taxonomy](https://arxiv.org/category_taxonomy).
 69    """
 70    links: list[Result.Link]
 71    """Up to three URLs associated with this result."""
 72    pdf_url: str | None
 73    """The URL of a PDF version of this result if present among links."""
 74
 75    def __init__(
 76        self,
 77        entry_id: str,
 78        updated: datetime = _DEFAULT_TIME,
 79        published: datetime = _DEFAULT_TIME,
 80        title: str = "",
 81        authors: list[Result.Author] | None = None,
 82        summary: str = "",
 83        comment: str | None = "",
 84        journal_ref: str | None = "",
 85        doi: str | None = "",
 86        primary_category: str = "",
 87        categories: list[str] | None = None,
 88        links: list[Result.Link] | None = None,
 89    ) -> None:
 90        """
 91        Constructs an arXiv search result item.
 92
 93        In most cases, results are produced by `Client.results`, which parses
 94        API responses internally.
 95        """
 96        self.entry_id = entry_id
 97        self.updated = updated
 98        self.published = published
 99        self.title = title
100        self.authors = authors or []
101        self.summary = summary
102        self.comment = comment
103        self.journal_ref = journal_ref
104        self.doi = doi
105        self.primary_category = primary_category
106        self.categories = categories or []
107        self.links = links or []
108        # Calculated members
109        self.pdf_url = Result._get_pdf_url(self.links)
110
111    def __str__(self) -> str:
112        return self.entry_id
113
114    def __repr__(self) -> str:
115        return (
116            "{}(entry_id={}, updated={}, published={}, title={}, authors={}, "
117            "summary={}, comment={}, journal_ref={}, doi={}, "
118            "primary_category={}, categories={}, links={})"
119        ).format(
120            _classname(self),
121            repr(self.entry_id),
122            repr(self.updated),
123            repr(self.published),
124            repr(self.title),
125            repr(self.authors),
126            repr(self.summary),
127            repr(self.comment),
128            repr(self.journal_ref),
129            repr(self.doi),
130            repr(self.primary_category),
131            repr(self.categories),
132            repr(self.links),
133        )
134
135    def __eq__(self, other: object) -> bool:
136        if isinstance(other, Result):
137            return self.entry_id == other.entry_id
138        return False
139
140    def get_short_id(self) -> str:
141        """
142        Returns the short ID for this result.
143
144        + If the result URL is `"https://arxiv.org/abs/2107.05580v1"`,
145        `result.get_short_id()` returns `2107.05580v1`.
146
147        + If the result URL is `"https://arxiv.org/abs/quant-ph/0201082v1"`,
148        `result.get_short_id()` returns `"quant-ph/0201082v1"` (the pre-March
149        2007 arXiv identifier format).
150
151        For an explanation of the difference between arXiv's legacy and current
152        identifiers, see [Understanding the arXiv
153        identifier](https://arxiv.org/help/arxiv_identifier).
154        """
155        return self.entry_id.split("arxiv.org/abs/")[-1]
156
157    def source_url(self) -> str | None:
158        """
159        Derives a URL for the source tarfile for this result.
160        """
161        if self.pdf_url is None:
162            return None
163        return self.pdf_url.replace("/pdf/", "/src/")
164
165    @staticmethod
166    def _get_pdf_url(links: list[Result.Link]) -> str | None:
167        """
168        Finds the PDF link among a result's links and returns its URL.
169
170        Should only be called once for a given `Result`, in its constructor.
171        After construction, the URL should be available in `Result.pdf_url`.
172        """
173        pdf_urls = [link.href for link in links if link.title == "pdf"]
174        if len(pdf_urls) == 0:
175            return None
176        elif len(pdf_urls) > 1:
177            logger.warning("Result has multiple PDF links; using %s", pdf_urls[0])
178        return pdf_urls[0]
179
180    @staticmethod
181    def _to_datetime(ts: time.struct_time) -> datetime:
182        """
183        Converts a UTC `time.struct_time` into a time-zone-aware `datetime`.
184
185        Retained as a stable utility for callers that historically relied on
186        feedparser's `*_parsed` time tuples; the internal Atom parser produces
187        `datetime` objects directly.
188        """
189        return datetime.fromtimestamp(timegm(ts), tz=timezone.utc)
190
191    class Author:
192        """
193        A light inner class for representing a result's authors.
194        """
195
196        name: str
197        """The author's name."""
198        affiliation: list[str]
199        """
200        Any `<arxiv:affiliation>` values associated with this author. Most
201        results have no affiliation data and this is an empty list; some
202        results have one or more affiliation strings per author.
203
204        See https://github.com/lukasschwab/arxiv.py/issues/62.
205        """
206
207        def __init__(self, name: str, affiliation: list[str] | None = None) -> None:
208            """
209            Constructs an `Author` with the specified name and (optional)
210            affiliations.
211            """
212            self.name = name
213            self.affiliation = affiliation or []
214
215        def __str__(self) -> str:
216            return self.name
217
218        def __repr__(self) -> str:
219            if self.affiliation:
220                return "{}({}, affiliation={})".format(
221                    _classname(self), repr(self.name), repr(self.affiliation)
222                )
223            return "{}({})".format(_classname(self), repr(self.name))
224
225        def __eq__(self, other: object) -> bool:
226            if isinstance(other, Result.Author):
227                return self.name == other.name
228            return False
229
230    class Link:
231        """
232        A light inner class for representing a result's links.
233        """
234
235        href: str
236        """The link's `href` attribute."""
237        title: str | None
238        """The link's title."""
239        rel: str
240        """The link's relationship to the `Result`."""
241        content_type: str | None
242        """The link's HTTP content type."""
243
244        def __init__(
245            self,
246            href: str,
247            title: str | None = None,
248            rel: str = "",
249            content_type: str | None = None,
250        ) -> None:
251            """
252            Constructs a `Link` with the specified link metadata.
253            """
254            self.href = href
255            self.title = title
256            self.rel = rel
257            self.content_type = content_type
258
259        def __str__(self) -> str:
260            return self.href
261
262        def __repr__(self) -> str:
263            return "{}({}, title={}, rel={}, content_type={})".format(
264                _classname(self),
265                repr(self.href),
266                repr(self.title),
267                repr(self.rel),
268                repr(self.content_type),
269            )
270
271        def __eq__(self, other: object) -> bool:
272            if isinstance(other, Result.Link):
273                return self.href == other.href
274            return False
275
276    class MissingFieldError(Exception):
277        """
278        An error indicating an entry is unparseable because it lacks required
279        fields.
280        """
281
282        missing_field: str
283        """The required field missing from the would-be entry."""
284        message: str
285        """Message describing what caused this error."""
286
287        def __init__(self, missing_field: str) -> None:
288            self.missing_field = missing_field
289            self.message = "Entry from arXiv missing required info"
290
291        def __repr__(self) -> str:
292            return "{}({})".format(_classname(self), repr(self.missing_field))
293
294
295class SortCriterion(Enum):
296    """
297    A SortCriterion identifies a property by which search results can be
298    sorted.
299
300    See [the arXiv API User's Manual: sort order for return
301    results](https://arxiv.org/help/api/user-manual#sort).
302    """
303
304    Relevance = "relevance"
305    LastUpdatedDate = "lastUpdatedDate"
306    SubmittedDate = "submittedDate"
307
308
309class SortOrder(Enum):
310    """
311    A SortOrder indicates order in which search results are sorted according
312    to the specified arxiv.SortCriterion.
313
314    See [the arXiv API User's Manual: sort order for return
315    results](https://arxiv.org/help/api/user-manual#sort).
316    """
317
318    Ascending = "ascending"
319    Descending = "descending"
320
321
322class Search:
323    """
324    A specification for a search of arXiv's database.
325
326    To run a search, use `Search.run` to use a default client or `Client.run`
327    with a specific client.
328    """
329
330    query: str
331    """
332    A query string.
333
334    This should be unencoded. Use `au:del_maestro AND ti:checkerboard`, not
335    `au:del_maestro+AND+ti:checkerboard`.
336
337    See [the arXiv API User's Manual: Details of Query
338    Construction](https://arxiv.org/help/api/user-manual#query_details).
339    """
340    id_list: list[str]
341    """
342    A list of arXiv article IDs to which to limit the search.
343
344    See [the arXiv API User's
345    Manual](https://arxiv.org/help/api/user-manual#search_query_and_id_list)
346    for documentation of the interaction between `query` and `id_list`.
347    """
348    max_results: int | None
349    """
350    The maximum number of results to be returned in an execution of this
351    search. To fetch every result available, set `max_results=None`.
352
353    The API's limit is 300,000 results per query.
354    """
355    sort_by: SortCriterion
356    """The sort criterion for results."""
357    sort_order: SortOrder
358    """The sort order for results."""
359
360    def __init__(
361        self,
362        query: str = "",
363        id_list: list[str] | None = None,
364        max_results: int | None = 100,
365        sort_by: SortCriterion = SortCriterion.Relevance,
366        sort_order: SortOrder = SortOrder.Descending,
367    ) -> None:
368        """
369        Constructs an arXiv API search with the specified criteria.
370        """
371        self.query = query
372        self.id_list = id_list or []
373        self.max_results = max_results
374        self.sort_by = sort_by
375        self.sort_order = sort_order
376
377    def __str__(self) -> str:
378        if self.query and self.id_list:
379            return f"Search(query='{self.query}', id_list={len(self.id_list)} items)"
380        elif self.query:
381            return f"Search(query='{self.query}')"
382        elif self.id_list:
383            return f"Search(id_list={len(self.id_list)} items)"
384        else:
385            return "Search(empty)"
386
387    def __repr__(self) -> str:
388        return ("{}(query={}, id_list={}, max_results={}, sort_by={}, sort_order={})").format(
389            _classname(self),
390            repr(self.query),
391            repr(self.id_list),
392            repr(self.max_results),
393            repr(self.sort_by),
394            repr(self.sort_order),
395        )
396
397    def _url_args(self) -> dict[str, str]:
398        """
399        Returns a dict of search parameters that should be included in an API
400        request for this search.
401        """
402        return {
403            "search_query": self.query,
404            "id_list": ",".join(self.id_list),
405            "sortBy": self.sort_by.value,
406            "sortOrder": self.sort_order.value,
407        }
408
409
410class Client:
411    """
412    Specifies a strategy for fetching results from arXiv's API.
413
414    This class obscures pagination and retry logic, and exposes
415    `Client.results`.
416    """
417
418    query_url_format = "https://export.arxiv.org/api/query?{}"
419    """
420    The arXiv query API endpoint format.
421    """
422    page_size: int
423    """
424    Maximum number of results fetched in a single API request. Smaller pages can
425    be retrieved faster, but may require more round-trips.
426
427    The API's limit is 2000 results per page.
428    """
429    delay_seconds: float
430    """
431    Number of seconds to wait between API requests.
432
433    [arXiv's Terms of Use](https://arxiv.org/help/api/tou) ask that you "make no
434    more than one request every three seconds."
435    """
436    num_retries: int
437    """
438    Number of times to retry a failing API request before raising an Exception.
439    """
440
441    _last_request_dt: datetime | None
442    _session: requests.Session
443
444    def __init__(
445        self, page_size: int = 100, delay_seconds: float = 3.0, num_retries: int = 3
446    ) -> None:
447        """
448        Constructs an arXiv API client with the specified options.
449
450        Note: the default parameters should provide a robust request strategy
451        for most use cases. Extreme page sizes, delays, or retries risk
452        violating the arXiv [API Terms of Use](https://arxiv.org/help/api/tou),
453        brittle behavior, and inconsistent results.
454        """
455        self.page_size = page_size
456        self.delay_seconds = delay_seconds
457        self.num_retries = num_retries
458        self._last_request_dt = None
459        self._session = requests.Session()
460
461    def __str__(self) -> str:
462        return f"Client(page_size={self.page_size}, delay={self.delay_seconds}s, retries={self.num_retries})"
463
464    def __repr__(self) -> str:
465        return "{}(page_size={}, delay_seconds={}, num_retries={})".format(
466            _classname(self),
467            repr(self.page_size),
468            repr(self.delay_seconds),
469            repr(self.num_retries),
470        )
471
472    def results(self, search: Search, offset: int = 0) -> Iterator[Result]:
473        """
474        Uses this client configuration to fetch one page of the search results
475        at a time, yielding the parsed `Result`s, until `max_results` results
476        have been yielded or there are no more search results.
477
478        If all tries fail, raises an `UnexpectedEmptyPageError` or `HTTPError`.
479
480        Setting a nonzero `offset` discards leading records in the result set.
481        When `offset` is greater than or equal to `search.max_results`, the full
482        result set is discarded.
483
484        For more on using generators, see
485        [Generators](https://wiki.python.org/moin/Generators).
486        """
487        limit = search.max_results - offset if search.max_results else None
488        if limit and limit < 0:
489            return iter(())
490        return itertools.islice(self._results(search, offset), limit)
491
492    def _results(self, search: Search, offset: int = 0) -> Generator[Result, None, None]:
493        page_url = self._format_url(search, offset, self.page_size)
494        feed = self._parse_feed(page_url, first_page=True)
495        if not feed.results:
496            logger.info("Got empty first page; stopping generation")
497            return
498        total_results = feed.header.total_results
499        logger.info(
500            "Got first page: %d of %d total results",
501            len(feed.results),
502            total_results,
503        )
504
505        while feed.results:
506            yield from feed.results
507            offset += len(feed.results)
508            if offset >= total_results:
509                break
510            page_url = self._format_url(search, offset, self.page_size)
511            feed = self._parse_feed(page_url, first_page=False)
512
513    def _format_url(self, search: Search, start: int, page_size: int) -> str:
514        """
515        Construct a request API for search that returns up to `page_size`
516        results starting with the result at index `start`.
517        """
518        url_args = search._url_args()
519        url_args.update(
520            {
521                "start": str(start),
522                "max_results": str(page_size),
523            }
524        )
525        return self.query_url_format.format(urlencode(url_args))
526
527    def _parse_feed(self, url: str, first_page: bool = True, _try_index: int = 0) -> ParsedFeed:
528        """
529        Fetches the specified URL and parses it as an Atom feed.
530
531        If a request fails or is unexpectedly empty, retries the request up to
532        `self.num_retries` times.
533        """
534        try:
535            return self.__try_parse_feed(url, first_page=first_page, try_index=_try_index)
536        except (
537            HTTPError,
538            UnexpectedEmptyPageError,
539            requests.exceptions.ConnectionError,
540        ) as err:
541            if _try_index < self.num_retries:
542                logger.debug("Got error (try %d): %s", _try_index, err)
543                return self._parse_feed(url, first_page=first_page, _try_index=_try_index + 1)
544            logger.debug("Giving up (try %d): %s", _try_index, err)
545            raise err
546
547    def __try_parse_feed(
548        self,
549        url: str,
550        first_page: bool,
551        try_index: int,
552    ) -> ParsedFeed:
553        """
554        Recursive helper for _parse_feed. Enforces `self.delay_seconds`: if that
555        number of seconds has not passed since `_parse_feed` was last called,
556        sleeps until delay_seconds seconds have passed.
557        """
558        # If this call would violate the rate limit, sleep until it doesn't.
559        if self._last_request_dt is not None:
560            required = timedelta(seconds=self.delay_seconds)
561            since_last_request = datetime.now() - self._last_request_dt
562            if since_last_request < required:
563                to_sleep = (required - since_last_request).total_seconds()
564                logger.info("Sleeping: %f seconds", to_sleep)
565                time.sleep(to_sleep)
566
567        logger.info("Requesting page (first: %r, try: %d): %s", first_page, try_index, url)
568
569        resp = self._session.get(url, headers={"user-agent": _USER_AGENT})
570        self._last_request_dt = datetime.now()
571        if resp.status_code != requests.codes.OK:
572            raise HTTPError(url, try_index, resp.status_code)
573
574        feed = _feed.parse(resp.content)
575        if len(feed.results) == 0 and not first_page:
576            raise UnexpectedEmptyPageError(url, try_index, feed)
577
578        if feed.malformed:
579            logger.warning("Malformed feed; consider handling: %s", feed.error)
580
581        return feed
582
583
584class ArxivError(Exception):
585    """This package's base Exception class."""
586
587    url: str
588    """The feed URL that could not be fetched."""
589    retry: int
590    """
591    The request try number which encountered this error; 0 for the initial try,
592    1 for the first retry, and so on.
593    """
594    message: str
595    """Message describing what caused this error."""
596
597    def __init__(self, url: str, retry: int, message: str) -> None:
598        """
599        Constructs an `ArxivError` encountered while fetching the specified URL.
600        """
601        self.url = url
602        self.retry = retry
603        self.message = message
604        super().__init__(self.message)
605
606    def __reduce__(self) -> tuple:
607        return (self.__class__, (self.url, self.retry, self.message))
608
609    def __str__(self) -> str:
610        return "{} ({})".format(self.message, self.url)
611
612
613class UnexpectedEmptyPageError(ArxivError):
614    """
615    An error raised when a page of results that should be non-empty is empty.
616
617    This should never happen in theory, but happens sporadically due to
618    brittleness in the underlying arXiv API; usually resolved by retries.
619
620    See `Client.results` for usage.
621    """
622
623    raw_feed: ParsedFeed
624    """
625    The raw parsed feed. Sometimes this contains useful diagnostic information,
626    e.g. in `bozo_exception`.
627    """
628
629    def __init__(self, url: str, retry: int, raw_feed: ParsedFeed) -> None:
630        """
631        Constructs an `UnexpectedEmptyPageError` encountered for the specified
632        API URL after `retry` tries.
633        """
634        self.url = url
635        self.raw_feed = raw_feed
636        super().__init__(url, retry, "Page of results was unexpectedly empty")
637
638    def __reduce__(self) -> tuple:
639        return (self.__class__, (self.url, self.retry, self.raw_feed))
640
641    def __repr__(self) -> str:
642        return "{}({}, {}, {})".format(
643            _classname(self), repr(self.url), repr(self.retry), repr(self.raw_feed)
644        )
645
646
647class HTTPError(ArxivError):
648    """
649    A non-200 status encountered while fetching a page of results.
650
651    See `Client.results` for usage.
652    """
653
654    status: int
655    """The HTTP status reported by the underlying request."""
656
657    def __init__(self, url: str, retry: int, status: int) -> None:
658        """
659        Constructs an `HTTPError` for the specified status code, encountered for
660        the specified API URL after `retry` tries.
661        """
662        self.url = url
663        self.status = status
664        super().__init__(
665            url,
666            retry,
667            "Page request resulted in HTTP {}".format(self.status),
668        )
669
670    def __reduce__(self) -> tuple:
671        return (self.__class__, (self.url, self.retry, self.status))
672
673    def __repr__(self) -> str:
674        return "{}({}, {}, {})".format(
675            _classname(self), repr(self.url), repr(self.retry), repr(self.status)
676        )
677
678
679def _classname(o: object) -> str:
680    """A helper function for use in __repr__ methods: arxiv.Result.Link."""
681    return "arxiv.{}".format(o.__class__.__qualname__)