Example config and check against repository issues to avoid duplicates
This commit is contained in:
Carlos Galindo
parent
fd915d92a3
commit
e0d09e5015
2 changed files with 149 additions and 4 deletions
116
config.py
Normal file
116
config.py
Normal file
|
|
@ -0,0 +1,116 @@
|
|||
'''
|
||||
The configuration file for issue_generator.
|
||||
Includes a custom reader for UDSClient, a shorthand custom reader and a
|
||||
shorthand custom poster for Forgejo.
|
||||
Two categories of project are given: software used on MIST and software
|
||||
packaged for ArchLinux by me.
|
||||
'''
|
||||
|
||||
import re
|
||||
from typing import Any
|
||||
import requests
|
||||
from issue_generator import FeedReader, GithubReader, GithubTagReader, PIPYReader, \
|
||||
IssuePoster, ForgejoPoster, GitlabPoster, \
|
||||
TIMEOUT
|
||||
import _secrets
|
||||
|
||||
|
||||
class UDSClientReader(FeedReader):
|
||||
'''Custom feed reader for UDSClient, whose version number appears in a .js file.'''
|
||||
|
||||
def __init__(self, target: IssuePoster):
|
||||
'''Creates a new UDSClient Reader.'''
|
||||
super().__init__("udsclient", "https://polilabs.upv.es/uds/utility/uds.js", target)
|
||||
|
||||
def read_feed(self) -> bool | None:
|
||||
'''Checks for a new version of udsclient, by checking a .js file.'''
|
||||
get = requests.get(self.url, timeout = TIMEOUT)
|
||||
if not self.target.check_req(get, self.name):
|
||||
return None
|
||||
match = re.search(r"[/a-zA-Z_-]+udsclient\d+-(\d+\.\d+\.\d+)\.tar\.gz", get.text)
|
||||
if not match:
|
||||
return None
|
||||
url = "https://polilabs.upv.es" + match.group(0)
|
||||
version = match.group(1)
|
||||
if requests.head(url, timeout = TIMEOUT).status_code != 200:
|
||||
return None
|
||||
return self.match_post_save(version, version, url)
|
||||
|
||||
def entry_get_link(self, entry: dict[str, Any]) -> str:
|
||||
raise NotImplementedError()
|
||||
|
||||
def entry_get_version(self, entry: dict[str, Any]) -> str:
|
||||
raise NotImplementedError()
|
||||
|
||||
|
||||
class CGJForgejoPoster(ForgejoPoster):
|
||||
'''All projects on Forgejo share the same assignee, instance and token.'''
|
||||
|
||||
def __init__(self, project: str):
|
||||
super().__init__(project, token = _secrets.FORGEJO_REPORTER_TOKEN,
|
||||
instance = "https://git.cgj.es")
|
||||
|
||||
|
||||
class NCAppReader(GithubReader):
|
||||
'''All GitHub releases readers that must alert Forgejo's `archpkgs`.'''
|
||||
def __init__(self, app, project):
|
||||
super().__init__(name = app, project = project,
|
||||
target = CGJForgejoPoster("archpkgs/" + app))
|
||||
|
||||
|
||||
# Issue Posters
|
||||
GITLAB_BOIRA_CARGAJI = GitlabPoster(project=36, assignee=4, labels="upstream-update",
|
||||
instance="https://mist.dsic.upv.es/git",
|
||||
token=_secrets.GITLAB_BOIRA_TOKEN)
|
||||
GITLAB_SNAKES_CARGAJI = GitlabPoster(project=37, assignee=4, labels="upstream-update",
|
||||
instance="https://mist.dsic.upv.es/git",
|
||||
token=_secrets.GITLAB_SNAKES_TOKEN)
|
||||
# Feed Readers
|
||||
FEED_READERS = [
|
||||
# Name FeedType Project TargetProject
|
||||
################################ Software used in MIST (Gitlab) ###################################
|
||||
# LanguageTool GHTags languagetool-org/languagetool 36 (sysadmin/boira)
|
||||
GithubTagReader(name = "LanguageTool", project = "languagetool-org/languagetool",
|
||||
target = GITLAB_BOIRA_CARGAJI),
|
||||
# python3-snakes PIPY snakes 37 (packages/python3-snakes)
|
||||
PIPYReader(name = "python3-snakes", package = "snakes",
|
||||
target = GITLAB_SNAKES_CARGAJI),
|
||||
################################ Software that I package (Forgejo) ################################
|
||||
# meshcentral GHReleases Ylianst/MeshCentral archpkgs/meshcentral
|
||||
GithubReader(name = "meshcentral", project = "Ylianst/MeshCentral",
|
||||
target = CGJForgejoPoster("archpkgs/meshcentral")),
|
||||
# nc-cospend GHReleases eneiluj/cospend-nc archpkgs/nextcloud-app-cospend
|
||||
NCAppReader(app = "nextcloud-app-cospend", project = "eneiluj/cospend-nc"),
|
||||
# nc-forms GHReleases nextcloud/forms archpkgs/nextcloud-app-forms
|
||||
NCAppReader(app = "nextcloud-app-forms", project = "nextcloud/forms"),
|
||||
# nc-maps GHReleases nextcloud/maps archpkgs/nextcloud-app-maps
|
||||
NCAppReader(app = "nextcloud-app-maps", project = "nextcloud/maps"),
|
||||
# nc-music GHReleases owncloud/music archpkgs/nextcloud-app-music
|
||||
NCAppReader(app = "nextcloud-app-music", project = "owncloud/music"),
|
||||
# nc-onlyoffice GHReleases ONLYOFFICE/onlyoffice-nextcloud archpkgs/nextcloud-app-onlyoffice
|
||||
NCAppReader(app = "nextcloud-app-onlyoffice", project = "ONLYOFFICE/onlyoffice-nextcloud"),
|
||||
# nc-polls GHReleases nextcloud/polls archpkgs/nextcloud-app-polls
|
||||
NCAppReader(app = "nextcloud-app-polls", project = "nextcloud/polls"),
|
||||
# nc-socialsharing GHReleases nextcloud/socialsharing archpkgs/nextcloud-app-socialsharing
|
||||
NCAppReader(app = "nextcloud-app-socialsharing", project = "nextcloud/socialsharing"),
|
||||
# udsclient Custom --- archpkgs/udsclient
|
||||
UDSClientReader(target = CGJForgejoPoster(project="archpkgs/udsclient")),
|
||||
# vigil GHReleases valeriansaliou/vigil archpkgs/vigil
|
||||
NCAppReader(app = "vigil", project = "valeriansaliou/vigil"),
|
||||
# vigil-local GHReleases valeriansaliou/vigil-local archpkgs/vigil-local
|
||||
NCAppReader(app = "vigil-local", project = "valeriansaliou/vigil-local"),
|
||||
# yourls GHReleases YOURLS/YOURLS archpkgs/yourls
|
||||
NCAppReader(app = "yourls", project = "YOURLS/YOURLS"),
|
||||
]
|
||||
|
||||
## PENDING:
|
||||
# Software that I use exposed to the Internet
|
||||
# Name FeedType Project
|
||||
# gad GHReleases brianreumere/gandi-automatic-dns aur?
|
||||
# nextcloud GHReleases nextcloud/server pacman?
|
||||
# peertube GHReleases Chocobozzz/PeerTube aur?
|
||||
# vaultwarden GHReleases dani-garcia/vaultwarden pacman?
|
||||
# wallabag GHReleases wallabag/wallabag pacman?
|
||||
# yay GHReleases Jguer/yay aur?
|
||||
# Others?
|
||||
# duplicati ???
|
||||
|
|
@ -29,6 +29,22 @@ class IssuePoster:
|
|||
'''Post a issue warning that version of software name has been published at link.'''
|
||||
raise NotImplementedError()
|
||||
|
||||
def issue_exists(self, name: str, version: str) -> bool:
|
||||
'''Checks whether the issue for the given name/version pair exists.'''
|
||||
expected_title = IssuePoster.title(name, version)
|
||||
get = requests.get(self.url,
|
||||
headers = self.auth_headers,
|
||||
timeout = TIMEOUT,
|
||||
params = self.search_params(expected_title))
|
||||
if not IssuePoster.check_req(get, name):
|
||||
return False
|
||||
data = get.json()
|
||||
return type(data) == list and any(issue['title'] == expected_title for issue in data)
|
||||
|
||||
def search_params(self, expected_title: str) -> dict[str, Any]:
|
||||
'''A parameter dictionary to query issues in different services.'''
|
||||
raise NotImplementedError()
|
||||
|
||||
@staticmethod
|
||||
def describe(name: str, version: str, link: str) -> str:
|
||||
'''Generate a description for the issue.'''
|
||||
|
|
@ -65,6 +81,7 @@ class GitlabPoster(IssuePoster):
|
|||
'''
|
||||
super().__init__(project, token, assignee, labels)
|
||||
self.api_url = f"{instance}/api/v4"
|
||||
self.url = f"{instance}/api/v4/projects/{project}/issues"
|
||||
self.auth_headers = { 'PRIVATE-TOKEN': self.token }
|
||||
|
||||
def post_issue(self, name: str, version: str, link: str) -> bool:
|
||||
|
|
@ -75,12 +92,15 @@ class GitlabPoster(IssuePoster):
|
|||
}
|
||||
if self.assignee:
|
||||
payload['assignee_id'] = self.assignee
|
||||
post = requests.post(f"{self.api_url}/projects/{self.project}/issues",
|
||||
post = requests.post(self.url,
|
||||
params = payload,
|
||||
headers = self.auth_headers,
|
||||
timeout = TIMEOUT)
|
||||
return IssuePoster.check_req(post, name)
|
||||
|
||||
def search_params(self, expected_title: str) -> bool:
|
||||
return { 'per_page': 1, 'search': expected_title, 'sort': 'asc', 'order_by': 'created_at' }
|
||||
|
||||
|
||||
class ForgejoPoster(IssuePoster):
|
||||
'''Forgejo (git.cgj.es instance) issue poster'''
|
||||
|
|
@ -101,6 +121,7 @@ class ForgejoPoster(IssuePoster):
|
|||
super().__init__(project, token, assignee, labels)
|
||||
self.auth_headers = { 'Authorization': f"token {self.token}" }
|
||||
self.api_url = f"{instance}/api/v1"
|
||||
self.url = f"{instance}/api/v1/repos/{project}/issues"
|
||||
|
||||
def post_issue(self, name: str, version: str, link: str) -> bool:
|
||||
payload = {
|
||||
|
|
@ -109,12 +130,15 @@ class ForgejoPoster(IssuePoster):
|
|||
"title": IssuePoster.title(name, version),
|
||||
"labels": self.labels if self.labels else [],
|
||||
}
|
||||
post = requests.post(f"{self.api_url}/repos/{self.project}/issues",
|
||||
post = requests.post(self.url,
|
||||
json = payload,
|
||||
headers = self.auth_headers,
|
||||
timeout = TIMEOUT)
|
||||
return IssuePoster.check_req(post, name)
|
||||
|
||||
def search_params(self, expected_title: str) -> bool:
|
||||
return { 'q': expected_title, 'state': 'all' }
|
||||
|
||||
def list_labels(self) -> dict[int, str] | None:
|
||||
'''Lists the labels and their IDs for this repository.'''
|
||||
get = requests.get(f"{self.api_url}/repos/{self.project}/labels",
|
||||
|
|
@ -166,6 +190,7 @@ class FeedReader:
|
|||
def match_post_save(self, version: str, _id: str, link: str) -> bool:
|
||||
'''Checks if a version is new, posts an issue and saves it as such.
|
||||
If the version is not new, or the posting fails, the method stops and returns False.'''
|
||||
# Match 1: with local file
|
||||
try:
|
||||
with open(CONFIG_DIR + self.name, encoding="utf-8") as file:
|
||||
match = file.readline().strip("\n") == str(_id)
|
||||
|
|
@ -173,8 +198,12 @@ class FeedReader:
|
|||
match = False
|
||||
if match:
|
||||
return False
|
||||
if not self.target.post_issue(self.name, version, link):
|
||||
return False
|
||||
# Match 2: with repository issues
|
||||
if not self.target.issue_exists(self.name, version):
|
||||
# Post the issue
|
||||
if not self.target.post_issue(self.name, version, link):
|
||||
return False
|
||||
# Save to disk
|
||||
if not os.path.isdir(CONFIG_DIR):
|
||||
os.makedirs(CONFIG_DIR)
|
||||
with open(CONFIG_DIR + self.name, mode="w", encoding="utf-8") as file:
|
||||
|
|
|
|||
Loading…
Reference in a new issue