From e0d09e501509e9c9db42bc1793010059e7f8adea Mon Sep 17 00:00:00 2001 From: Carlos Galindo Date: Thu, 7 Sep 2023 17:40:13 +0200 Subject: [PATCH] Example config and check against repository issues to avoid duplicates --- config.py | 116 +++++++++++++++++++++++++++++++++++++++++++++ issue_generator.py | 37 +++++++++++++-- 2 files changed, 149 insertions(+), 4 deletions(-) create mode 100644 config.py diff --git a/config.py b/config.py new file mode 100644 index 0000000..23b0569 --- /dev/null +++ b/config.py @@ -0,0 +1,116 @@ +''' +The configuration file for issue_generator. +Includes a custom reader for UDSClient, a shorthand custom reader and a +shorthand custom poster for Forgejo. +Two categories of project are given: software used on MIST and software +packaged for ArchLinux by me. +''' + +import re +from typing import Any +import requests +from issue_generator import FeedReader, GithubReader, GithubTagReader, PIPYReader, \ + IssuePoster, ForgejoPoster, GitlabPoster, \ + TIMEOUT +import _secrets + + +class UDSClientReader(FeedReader): + '''Custom feed reader for UDSClient, whose version number appears in a .js file.''' + + def __init__(self, target: IssuePoster): + '''Creates a new UDSClient Reader.''' + super().__init__("udsclient", "https://polilabs.upv.es/uds/utility/uds.js", target) + + def read_feed(self) -> bool | None: + '''Checks for a new version of udsclient, by checking a .js file.''' + get = requests.get(self.url, timeout = TIMEOUT) + if not self.target.check_req(get, self.name): + return None + match = re.search(r"[/a-zA-Z_-]+udsclient\d+-(\d+\.\d+\.\d+)\.tar\.gz", get.text) + if not match: + return None + url = "https://polilabs.upv.es" + match.group(0) + version = match.group(1) + if requests.head(url, timeout = TIMEOUT).status_code != 200: + return None + return self.match_post_save(version, version, url) + + def entry_get_link(self, entry: dict[str, Any]) -> str: + raise NotImplementedError() + + def entry_get_version(self, entry: dict[str, Any]) -> str: + raise NotImplementedError() + + +class CGJForgejoPoster(ForgejoPoster): + '''All projects on Forgejo share the same assignee, instance and token.''' + + def __init__(self, project: str): + super().__init__(project, token = _secrets.FORGEJO_REPORTER_TOKEN, + instance = "https://git.cgj.es") + + +class NCAppReader(GithubReader): + '''All GitHub releases readers that must alert Forgejo's `archpkgs`.''' + def __init__(self, app, project): + super().__init__(name = app, project = project, + target = CGJForgejoPoster("archpkgs/" + app)) + + +# Issue Posters +GITLAB_BOIRA_CARGAJI = GitlabPoster(project=36, assignee=4, labels="upstream-update", + instance="https://mist.dsic.upv.es/git", + token=_secrets.GITLAB_BOIRA_TOKEN) +GITLAB_SNAKES_CARGAJI = GitlabPoster(project=37, assignee=4, labels="upstream-update", + instance="https://mist.dsic.upv.es/git", + token=_secrets.GITLAB_SNAKES_TOKEN) +# Feed Readers +FEED_READERS = [ +# Name FeedType Project TargetProject +################################ Software used in MIST (Gitlab) ################################### +# LanguageTool GHTags languagetool-org/languagetool 36 (sysadmin/boira) + GithubTagReader(name = "LanguageTool", project = "languagetool-org/languagetool", + target = GITLAB_BOIRA_CARGAJI), +# python3-snakes PIPY snakes 37 (packages/python3-snakes) + PIPYReader(name = "python3-snakes", package = "snakes", + target = GITLAB_SNAKES_CARGAJI), +################################ Software that I package (Forgejo) ################################ +# meshcentral GHReleases Ylianst/MeshCentral archpkgs/meshcentral + GithubReader(name = "meshcentral", project = "Ylianst/MeshCentral", + target = CGJForgejoPoster("archpkgs/meshcentral")), +# nc-cospend GHReleases eneiluj/cospend-nc archpkgs/nextcloud-app-cospend + NCAppReader(app = "nextcloud-app-cospend", project = "eneiluj/cospend-nc"), +# nc-forms GHReleases nextcloud/forms archpkgs/nextcloud-app-forms + NCAppReader(app = "nextcloud-app-forms", project = "nextcloud/forms"), +# nc-maps GHReleases nextcloud/maps archpkgs/nextcloud-app-maps + NCAppReader(app = "nextcloud-app-maps", project = "nextcloud/maps"), +# nc-music GHReleases owncloud/music archpkgs/nextcloud-app-music + NCAppReader(app = "nextcloud-app-music", project = "owncloud/music"), +# nc-onlyoffice GHReleases ONLYOFFICE/onlyoffice-nextcloud archpkgs/nextcloud-app-onlyoffice + NCAppReader(app = "nextcloud-app-onlyoffice", project = "ONLYOFFICE/onlyoffice-nextcloud"), +# nc-polls GHReleases nextcloud/polls archpkgs/nextcloud-app-polls + NCAppReader(app = "nextcloud-app-polls", project = "nextcloud/polls"), +# nc-socialsharing GHReleases nextcloud/socialsharing archpkgs/nextcloud-app-socialsharing + NCAppReader(app = "nextcloud-app-socialsharing", project = "nextcloud/socialsharing"), +# udsclient Custom --- archpkgs/udsclient + UDSClientReader(target = CGJForgejoPoster(project="archpkgs/udsclient")), +# vigil GHReleases valeriansaliou/vigil archpkgs/vigil + NCAppReader(app = "vigil", project = "valeriansaliou/vigil"), +# vigil-local GHReleases valeriansaliou/vigil-local archpkgs/vigil-local + NCAppReader(app = "vigil-local", project = "valeriansaliou/vigil-local"), +# yourls GHReleases YOURLS/YOURLS archpkgs/yourls + NCAppReader(app = "yourls", project = "YOURLS/YOURLS"), +] + +## PENDING: +# Software that I use exposed to the Internet +# Name FeedType Project +# gad GHReleases brianreumere/gandi-automatic-dns aur? +# nextcloud GHReleases nextcloud/server pacman? +# peertube GHReleases Chocobozzz/PeerTube aur? +# vaultwarden GHReleases dani-garcia/vaultwarden pacman? +# wallabag GHReleases wallabag/wallabag pacman? +# yay GHReleases Jguer/yay aur? +# Others? +# duplicati ??? diff --git a/issue_generator.py b/issue_generator.py index 00d0cc3..0ebc0dc 100755 --- a/issue_generator.py +++ b/issue_generator.py @@ -29,6 +29,22 @@ class IssuePoster: '''Post a issue warning that version of software name has been published at link.''' raise NotImplementedError() + def issue_exists(self, name: str, version: str) -> bool: + '''Checks whether the issue for the given name/version pair exists.''' + expected_title = IssuePoster.title(name, version) + get = requests.get(self.url, + headers = self.auth_headers, + timeout = TIMEOUT, + params = self.search_params(expected_title)) + if not IssuePoster.check_req(get, name): + return False + data = get.json() + return type(data) == list and any(issue['title'] == expected_title for issue in data) + + def search_params(self, expected_title: str) -> dict[str, Any]: + '''A parameter dictionary to query issues in different services.''' + raise NotImplementedError() + @staticmethod def describe(name: str, version: str, link: str) -> str: '''Generate a description for the issue.''' @@ -65,6 +81,7 @@ class GitlabPoster(IssuePoster): ''' super().__init__(project, token, assignee, labels) self.api_url = f"{instance}/api/v4" + self.url = f"{instance}/api/v4/projects/{project}/issues" self.auth_headers = { 'PRIVATE-TOKEN': self.token } def post_issue(self, name: str, version: str, link: str) -> bool: @@ -75,12 +92,15 @@ class GitlabPoster(IssuePoster): } if self.assignee: payload['assignee_id'] = self.assignee - post = requests.post(f"{self.api_url}/projects/{self.project}/issues", + post = requests.post(self.url, params = payload, headers = self.auth_headers, timeout = TIMEOUT) return IssuePoster.check_req(post, name) + def search_params(self, expected_title: str) -> bool: + return { 'per_page': 1, 'search': expected_title, 'sort': 'asc', 'order_by': 'created_at' } + class ForgejoPoster(IssuePoster): '''Forgejo (git.cgj.es instance) issue poster''' @@ -101,6 +121,7 @@ class ForgejoPoster(IssuePoster): super().__init__(project, token, assignee, labels) self.auth_headers = { 'Authorization': f"token {self.token}" } self.api_url = f"{instance}/api/v1" + self.url = f"{instance}/api/v1/repos/{project}/issues" def post_issue(self, name: str, version: str, link: str) -> bool: payload = { @@ -109,12 +130,15 @@ class ForgejoPoster(IssuePoster): "title": IssuePoster.title(name, version), "labels": self.labels if self.labels else [], } - post = requests.post(f"{self.api_url}/repos/{self.project}/issues", + post = requests.post(self.url, json = payload, headers = self.auth_headers, timeout = TIMEOUT) return IssuePoster.check_req(post, name) + def search_params(self, expected_title: str) -> bool: + return { 'q': expected_title, 'state': 'all' } + def list_labels(self) -> dict[int, str] | None: '''Lists the labels and their IDs for this repository.''' get = requests.get(f"{self.api_url}/repos/{self.project}/labels", @@ -166,6 +190,7 @@ class FeedReader: def match_post_save(self, version: str, _id: str, link: str) -> bool: '''Checks if a version is new, posts an issue and saves it as such. If the version is not new, or the posting fails, the method stops and returns False.''' + # Match 1: with local file try: with open(CONFIG_DIR + self.name, encoding="utf-8") as file: match = file.readline().strip("\n") == str(_id) @@ -173,8 +198,12 @@ class FeedReader: match = False if match: return False - if not self.target.post_issue(self.name, version, link): - return False + # Match 2: with repository issues + if not self.target.issue_exists(self.name, version): + # Post the issue + if not self.target.post_issue(self.name, version, link): + return False + # Save to disk if not os.path.isdir(CONFIG_DIR): os.makedirs(CONFIG_DIR) with open(CONFIG_DIR + self.name, mode="w", encoding="utf-8") as file: