diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2ad25334..4eaafccd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -25,9 +25,10 @@ Here are some links to the documentation that could be helpful when contributing - for details also see [official GitLab API docs](https://docs.gitlab.com/ee/api/) - Pagure (through `requests`) - API is dependent on deployed version of Pagure service; `ogr` is majorly used on (links lead directly to API docs) - - [src.fedoraproject.org](https://src.fedoraproject.org/api/0/) - [pagure.io](https://pagure.io/api/0/) - [git.stg.centos.org](https://git.stg.centos.org/api/0/) +- Forgejo + - [src.fedoraproject.org](https://src.fedoraproject.org/api/v1/) ## Making raw HTTP requests diff --git a/ogr/constant.py b/ogr/constant.py index 6a432f9e..87d8ff09 100644 --- a/ogr/constant.py +++ b/ogr/constant.py @@ -3,3 +3,10 @@ CLONE_TIMEOUT = 60 DEFAULT_RO_PREFIX_STRING = "READ ONLY: " + +DGIT_URLS = ( + "src.fedoraproject.org", + "src.stg.fedoraproject.org", + "pkgs.fedoraproject.org", + "pkgs.stg.fedoraproject.org", +) diff --git a/ogr/factory.py b/ogr/factory.py index e2ccc154..be28ba53 100644 --- a/ogr/factory.py +++ b/ogr/factory.py @@ -2,15 +2,26 @@ # SPDX-License-Identifier: MIT import functools +import logging +import time from collections.abc import Iterable from typing import Optional +from requests.exceptions import ConnectionError, ReadTimeout + from ogr.abstract import GitProject, GitService -from ogr.exceptions import OgrException +from ogr.constant import DGIT_URLS +from ogr.exceptions import OgrException, OgrNetworkError from ogr.parsing import parse_git_repo _SERVICE_MAPPING: dict[str, type[GitService]] = {} +# cache dist-git forge service class for two minutes +_DGIT_FORGE_CACHE: dict[str, tuple[float, type[GitService]]] = {} +_DGIT_FORGE_CACHE_TTL = 120 + +logger = logging.getLogger(__name__) + def use_for_service(service: str, _func=None): """ @@ -29,7 +40,7 @@ class GithubService(BaseGitService): pass @use_for_service("pagure.io") - @use_for_service("src.fedoraproject.org") + @use_for_service("git.centos.org") class PagureService(BaseGitService): pass ``` @@ -116,6 +127,10 @@ def get_service_class_or_none( ) -> Optional[type[GitService]]: """ Get the matching service class from the URL. + When attempting to get the matching service class for dist-git, probing + is used to determine whether `PagureService` or `ForgejoService` + should be returned. This information is cached for two minutes. The + probing request is set to timeout after 5 seconds. Args: url: URL of the project, e.g. `"https://github.com/packit/ogr"`. @@ -126,13 +141,59 @@ def get_service_class_or_none( Returns: Matched class (subclass of `GitService`) or `None`. + + Raises: + OgrNetworkError, in case a ConnectionError or ReadTimeout error + is encountered when attempting to probe Pagure dist-git. """ mapping = {} mapping.update(_SERVICE_MAPPING) + non_overridden_dgit_urls: Iterable[str] = DGIT_URLS + if service_mapping_update: mapping.update(service_mapping_update) + non_overridden_dgit_urls = ( + set(non_overridden_dgit_urls) - service_mapping_update.keys() + ) parsed_url = parse_git_repo(url) + + # [XXX] remove once the migration of dist-git is finished + for dgit_url in non_overridden_dgit_urls: + + # if dealing with dist-git, we need to check whether we need to use + # `PagureService` or `ForgejoService` + if dgit_url in parsed_url.hostname: + + from ogr.services.forgejo import ForgejoService + from ogr.services.pagure import PagureService + + now = time.monotonic() + if cache := _DGIT_FORGE_CACHE.get(dgit_url): + timestamp, service_type = cache + if now - timestamp < _DGIT_FORGE_CACHE_TTL: + return service_type + # API call to the Pagure backend + api_endpoint = "https://src.fedoraproject.org/api/0/version" + api_endpoint_stg = "https://src.stg.fedoraproject.org/api/0/version" + request_url = api_endpoint_stg if ".stg." in dgit_url else api_endpoint + + try: + pagure_service = PagureService() + response = pagure_service.get_raw_request(url=request_url, timeout=5) + + # if not found, then dist-git is no longer hosted on Pagure + dgit_service_kls = ( + PagureService if response.status_code != 404 else ForgejoService + ) + + _DGIT_FORGE_CACHE[dgit_url] = (now, dgit_service_kls) + return dgit_service_kls + + except (ConnectionError, ReadTimeout) as er: + logger.error(er) + raise OgrNetworkError(f"Cannot connect to url: '{url}'.") from er + for service, service_kls in mapping.items(): if parse_git_repo(service).hostname in parsed_url.hostname: return service_kls @@ -178,13 +239,13 @@ def get_instances_from_dict(instances: dict) -> set[GitService]: ```py get_instances_from_dict({ "github.com": {"token": "abcd"}, - "pagure": { + "forgejo": { "token": "abcd", "instance_url": "https://src.fedoraproject.org", }, }) == { GithubService(token="abcd"), - PagureService(token="abcd", instance_url="https://src.fedoraproject.org") + ForgejoService(token="abcd", instance_url="https://src.fedoraproject.org") } ``` diff --git a/ogr/services/forgejo/service.py b/ogr/services/forgejo/service.py index c85fb628..0a1e473a 100644 --- a/ogr/services/forgejo/service.py +++ b/ogr/services/forgejo/service.py @@ -17,6 +17,10 @@ @use_for_service("forgejo") @use_for_service("codeberg.org") +@use_for_service("src.fedoraproject.org") +@use_for_service("src.stg.fedoraproject.org") +@use_for_service("pkgs.fedoraproject.org") +@use_for_service("pkgs.stg.fedoraproject.org") class ForgejoService(BaseGitService): version = "/api/v1" diff --git a/ogr/services/pagure/project.py b/ogr/services/pagure/project.py index cea9d424..3284c5e1 100644 --- a/ogr/services/pagure/project.py +++ b/ogr/services/pagure/project.py @@ -382,8 +382,6 @@ def is_private(self) -> bool: "git.centos.org", "git.stg.centos.org", "pagure.io", - "src.fedoraproject.org", - "src.stg.fedoraproject.org", ]: # private repositories are not allowed on generally used pagure instances return False diff --git a/ogr/services/pagure/service.py b/ogr/services/pagure/service.py index 354a6ac2..3bd8b5c9 100644 --- a/ogr/services/pagure/service.py +++ b/ogr/services/pagure/service.py @@ -26,17 +26,13 @@ @use_for_service("pagure") -@use_for_service("src.fedoraproject.org") -@use_for_service("src.stg.fedoraproject.org") -@use_for_service("pkgs.fedoraproject.org") -@use_for_service("pkgs.stg.fedoraproject.org") @use_for_service("git.centos.org") @use_for_service("git.stg.centos.org") class PagureService(BaseGitService): def __init__( self, token: Optional[str] = None, - instance_url: str = "https://src.fedoraproject.org", + instance_url: str = "https://pagure.io", read_only: bool = False, insecure: bool = False, max_retries: Union[int, urllib3.util.Retry] = 3, @@ -250,6 +246,7 @@ def get_raw_request( params=None, data=None, header=None, + timeout: Optional[float] = None, ) -> RequestResponse: """ Call API endpoint and wrap the response in `RequestResponse` type. @@ -279,6 +276,7 @@ def get_raw_request( headers=headers, data=data, verify=not self.insecure, + timeout=timeout, ) logger.debug( f"Ogr sent request with following headers: {headers | {'Authorization': ''}}", diff --git a/tests/integration/factory/test_factory.py b/tests/integration/factory/test_factory.py index 4cb5b537..ddd4b280 100644 --- a/tests/integration/factory/test_factory.py +++ b/tests/integration/factory/test_factory.py @@ -48,7 +48,10 @@ def github_service(self): @property def pagure_service(self): if not self._pagure_service: - self._pagure_service = PagureService(token=self.pagure_token) + self._pagure_service = PagureService( + token=self.pagure_token, + instance_url="https://src.fedoraproject.org", + ) return self._pagure_service @property diff --git a/tests/integration/pagure/test_data/test_project_token/PagureProjectTokenCommands.test_is_private.yaml b/tests/integration/pagure/test_data/test_project_token/PagureProjectTokenCommands.test_is_private.yaml index 72742c59..8a5fa939 100644 --- a/tests/integration/pagure/test_data/test_project_token/PagureProjectTokenCommands.test_is_private.yaml +++ b/tests/integration/pagure/test_data/test_project_token/PagureProjectTokenCommands.test_is_private.yaml @@ -5,7 +5,7 @@ _requre: requests.sessions: send: POST: - https://src.fedoraproject.org/api/0/-/whoami: + https://pagure.io/api/0/-/whoami: - metadata: latency: 0.7874109745025635 module_call_list: diff --git a/tests/integration/pagure/test_project_token.py b/tests/integration/pagure/test_project_token.py index d57886f6..e52c5179 100644 --- a/tests/integration/pagure/test_project_token.py +++ b/tests/integration/pagure/test_project_token.py @@ -195,7 +195,7 @@ def test_pr_status(self): ) def test_is_private(self): - self.service.instance_url = "https://src.fedoraproject.org" + self.service.instance_url = "https://pagure.io" assert not self.ogr_project.is_private() def test_token_is_none_then_set(self): diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py new file mode 100644 index 00000000..d4457d94 --- /dev/null +++ b/tests/unit/conftest.py @@ -0,0 +1,29 @@ +# Copyright Contributors to the Packit project. +# SPDX-License-Identifier: MIT + +import pytest +from flexmock import flexmock + +from ogr import PagureService + + +# mocks API calls to Pagure dist-git made to determine whether dist-git +# is still hosted on Pagure and returns the status code expected after +# the migration of dist-git to Forgejo +@pytest.fixture(autouse=True) +def setup_api_request_mock(): + response = flexmock(status_code=404) + flexmock(PagureService).should_receive("get_raw_request").with_args( + url="https://src.fedoraproject.org/api/0/version", + timeout=5, + ).and_return( + response, + ) + flexmock(PagureService).should_receive("get_raw_request").with_args( + url="https://src.stg.fedoraproject.org/api/0/version", + timeout=5, + ).and_return( + response, + ) + + return diff --git a/tests/unit/test_factory.py b/tests/unit/test_factory.py index f08cf462..b047d1a7 100644 --- a/tests/unit/test_factory.py +++ b/tests/unit/test_factory.py @@ -6,9 +6,10 @@ from flexmock import Mock, flexmock from urllib3.util import Retry -from ogr import GithubService, GitlabService, PagureService +from ogr import ForgejoService, GithubService, GitlabService, PagureService from ogr.exceptions import OgrException from ogr.factory import get_instances_from_dict, get_project, get_service_class +from ogr.services.forgejo import ForgejoProject from ogr.services.github import GithubProject from ogr.services.gitlab import GitlabProject from ogr.services.pagure import PagureProject @@ -35,7 +36,7 @@ {"github.com": PagureService}, PagureService, ), - ("https://src.fedoraproject.org/rpms/python-ogr", None, PagureService), + ("https://src.fedoraproject.org/rpms/python-ogr", None, ForgejoService), ("https://pagure.io/ogr", None, PagureService), ("https://pagure.something.com/ogr", None, PagureService), ("https://gitlab.com/someone/project", None, GitlabService), @@ -55,18 +56,18 @@ ( "https://src.fedoraproject.org/rpms/golang-gitlab-flimzy-testy", None, - PagureService, + ForgejoService, ), ( "https://src.stg.fedoraproject.org/rpms/golang-gitlab-flimzy-testy", None, - PagureService, + ForgejoService, ), - ("https://src.fedoraproject.org/rpms/python-gitlab", None, PagureService), + ("https://src.fedoraproject.org/rpms/python-gitlab", None, ForgejoService), ( "https://src.fedoraproject.org/rpms/golang-gitlab-yawning-utls", None, - PagureService, + ForgejoService, ), ], ) @@ -165,10 +166,10 @@ def test_get_service_class_not_found(url, mapping): None, None, True, - PagureProject( + ForgejoProject( namespace="rpms", repo="python-ogr", - service=PagureService(instance_url="https://src.fedoraproject.org"), + service=ForgejoService(instance_url="https://src.fedoraproject.org"), ), ), ( @@ -350,6 +351,7 @@ def test_get_project_not_found(url, mapping, instances, exc_str): ({"github.com": {"token": "abcd"}}, {GithubService(token="abcd")}), ({"gitlab": {"token": "abcd"}}, {GitlabService(token="abcd")}), ({"pagure": {"token": "abcd"}}, {PagureService(token="abcd")}), + ({"forgejo": {"token": "abcd"}}, {ForgejoService(token="abcd")}), ( { "pagure": { @@ -359,6 +361,20 @@ def test_get_project_not_found(url, mapping, instances, exc_str): }, {PagureService(token="abcd", instance_url="https://src.fedoraproject.org")}, ), + ( + { + "forgejo": { + "token": "abcd", + "instance_url": "https://src.fedoraproject.org", + }, + }, + { + ForgejoService( + token="abcd", + instance_url="https://src.fedoraproject.org", + ), + }, + ), ( {"github.com": {"token": "abcd"}, "gitlab": {"token": "abcd"}}, {GithubService(token="abcd"), GitlabService(token="abcd")}, diff --git a/tests/unit/test_pagure.py b/tests/unit/test_pagure.py index 7650a186..2dc122e0 100644 --- a/tests/unit/test_pagure.py +++ b/tests/unit/test_pagure.py @@ -8,5 +8,8 @@ class TestPagureService(TestCase): def test_hostname(self): - assert PagureService().hostname == "src.fedoraproject.org" - assert PagureService(instance_url="https://pagure.io").hostname == "pagure.io" + assert PagureService().hostname == "pagure.io" + assert ( + PagureService(instance_url="https://git.centos.org").hostname + == "git.centos.org" + )