Example config and check against repository issues to avoid duplicates

config.py

@@ -0,0 +1,116 @@
+'''
+The configuration file for issue_generator.
+Includes a custom reader for UDSClient, a shorthand custom reader and a
+shorthand custom poster for Forgejo.
+Two categories of project are given: software used on MIST and software
+packaged for ArchLinux by me.
+'''
+
+import re
+from typing import Any
+import requests
+from issue_generator import FeedReader, GithubReader, GithubTagReader, PIPYReader, \
+        IssuePoster, ForgejoPoster, GitlabPoster, \
+        TIMEOUT
+import _secrets
+
+
+class UDSClientReader(FeedReader):
+    '''Custom feed reader for UDSClient, whose version number appears in a .js file.'''
+
+    def __init__(self, target: IssuePoster):
+        '''Creates a new UDSClient Reader.'''
+        super().__init__("udsclient", "https://polilabs.upv.es/uds/utility/uds.js", target)
+
+    def read_feed(self) -> bool | None:
+        '''Checks for a new version of udsclient, by checking a .js file.'''
+        get = requests.get(self.url, timeout = TIMEOUT)
+        if not self.target.check_req(get, self.name):
+            return None
+        match = re.search(r"[/a-zA-Z_-]+udsclient\d+-(\d+\.\d+\.\d+)\.tar\.gz", get.text)
+        if not match:
+            return None
+        url = "https://polilabs.upv.es" + match.group(0)
+        version = match.group(1)
+        if requests.head(url, timeout = TIMEOUT).status_code != 200:
+            return None
+        return self.match_post_save(version, version, url)
+
+    def entry_get_link(self, entry: dict[str, Any]) -> str:
+        raise NotImplementedError()
+
+    def entry_get_version(self, entry: dict[str, Any]) -> str:
+        raise NotImplementedError()
+
+
+class CGJForgejoPoster(ForgejoPoster):
+    '''All projects on Forgejo share the same assignee, instance and token.'''
+
+    def __init__(self, project: str):
+        super().__init__(project, token = _secrets.FORGEJO_REPORTER_TOKEN,
+                         instance = "https://git.cgj.es")
+
+
+class NCAppReader(GithubReader):
+    '''All GitHub releases readers that must alert Forgejo's `archpkgs`.'''
+    def __init__(self, app, project):
+        super().__init__(name = app, project = project,
+                         target = CGJForgejoPoster("archpkgs/" + app))
+
+
+# Issue Posters
+GITLAB_BOIRA_CARGAJI  = GitlabPoster(project=36, assignee=4, labels="upstream-update",
+                                     instance="https://mist.dsic.upv.es/git",
+                                     token=_secrets.GITLAB_BOIRA_TOKEN)
+GITLAB_SNAKES_CARGAJI = GitlabPoster(project=37, assignee=4, labels="upstream-update",
+                                     instance="https://mist.dsic.upv.es/git",
+                                     token=_secrets.GITLAB_SNAKES_TOKEN)
+# Feed Readers
+FEED_READERS = [
+# Name             FeedType   Project                         TargetProject
+################################ Software used in MIST (Gitlab) ###################################
+# LanguageTool     GHTags     languagetool-org/languagetool   36 (sysadmin/boira)
+        GithubTagReader(name   = "LanguageTool", project = "languagetool-org/languagetool",
+                        target = GITLAB_BOIRA_CARGAJI),
+# python3-snakes   PIPY       snakes                          37 (packages/python3-snakes)
+        PIPYReader(name   = "python3-snakes", package = "snakes",
+                   target = GITLAB_SNAKES_CARGAJI),
+################################ Software that I package (Forgejo) ################################
+# meshcentral      GHReleases Ylianst/MeshCentral             archpkgs/meshcentral
+        GithubReader(name   = "meshcentral", project = "Ylianst/MeshCentral",
+                     target = CGJForgejoPoster("archpkgs/meshcentral")),
+# nc-cospend       GHReleases eneiluj/cospend-nc              archpkgs/nextcloud-app-cospend
+        NCAppReader(app = "nextcloud-app-cospend", project = "eneiluj/cospend-nc"),
+# nc-forms         GHReleases nextcloud/forms                 archpkgs/nextcloud-app-forms
+        NCAppReader(app = "nextcloud-app-forms", project = "nextcloud/forms"),
+# nc-maps          GHReleases nextcloud/maps                  archpkgs/nextcloud-app-maps
+        NCAppReader(app = "nextcloud-app-maps", project = "nextcloud/maps"),
+# nc-music         GHReleases owncloud/music                  archpkgs/nextcloud-app-music
+        NCAppReader(app = "nextcloud-app-music", project = "owncloud/music"),
+# nc-onlyoffice    GHReleases ONLYOFFICE/onlyoffice-nextcloud archpkgs/nextcloud-app-onlyoffice
+        NCAppReader(app = "nextcloud-app-onlyoffice", project = "ONLYOFFICE/onlyoffice-nextcloud"),
+# nc-polls         GHReleases nextcloud/polls                 archpkgs/nextcloud-app-polls
+        NCAppReader(app = "nextcloud-app-polls", project = "nextcloud/polls"),
+# nc-socialsharing GHReleases nextcloud/socialsharing         archpkgs/nextcloud-app-socialsharing
+        NCAppReader(app = "nextcloud-app-socialsharing", project = "nextcloud/socialsharing"),
+# udsclient        Custom     ---                             archpkgs/udsclient
+        UDSClientReader(target = CGJForgejoPoster(project="archpkgs/udsclient")),
+# vigil            GHReleases valeriansaliou/vigil            archpkgs/vigil
+        NCAppReader(app = "vigil", project = "valeriansaliou/vigil"),
+# vigil-local      GHReleases valeriansaliou/vigil-local      archpkgs/vigil-local
+        NCAppReader(app = "vigil-local", project = "valeriansaliou/vigil-local"),
+# yourls           GHReleases YOURLS/YOURLS                   archpkgs/yourls
+        NCAppReader(app = "yourls", project = "YOURLS/YOURLS"),
+]
+
+## PENDING:
+# Software that I use exposed to the Internet
+# Name             FeedType   Project
+# gad              GHReleases brianreumere/gandi-automatic-dns aur?
+# nextcloud        GHReleases nextcloud/server                 pacman?
+# peertube         GHReleases Chocobozzz/PeerTube              aur?
+# vaultwarden      GHReleases dani-garcia/vaultwarden          pacman?
+# wallabag         GHReleases wallabag/wallabag                pacman?
+# yay              GHReleases Jguer/yay                        aur?
+# Others?
+# duplicati ???

issue_generator.py

@@ -29,6 +29,22 @@ class IssuePoster:
         '''Post a issue warning that version of software name has been published at link.'''
         raise NotImplementedError()
 
+    def issue_exists(self, name: str, version: str) -> bool:
+        '''Checks whether the issue for the given name/version pair exists.'''
+        expected_title = IssuePoster.title(name, version)
+        get = requests.get(self.url,
+                           headers = self.auth_headers,
+                           timeout = TIMEOUT,
+                           params = self.search_params(expected_title))
+        if not IssuePoster.check_req(get, name):
+            return False
+        data = get.json()
+        return type(data) == list and any(issue['title'] == expected_title for issue in data)
+
+    def search_params(self, expected_title: str) -> dict[str, Any]:
+        '''A parameter dictionary to query issues in different services.'''
+        raise NotImplementedError()
+
     @staticmethod
     def describe(name: str, version: str, link: str) -> str:
         '''Generate a description for the issue.'''
@@ -65,6 +81,7 @@ class GitlabPoster(IssuePoster):
         '''
         super().__init__(project, token, assignee, labels)
         self.api_url = f"{instance}/api/v4"
+        self.url = f"{instance}/api/v4/projects/{project}/issues"
         self.auth_headers = { 'PRIVATE-TOKEN': self.token }
 
     def post_issue(self, name: str, version: str, link: str) -> bool:
@@ -75,12 +92,15 @@ class GitlabPoster(IssuePoster):
         }
         if self.assignee:
             payload['assignee_id'] = self.assignee
-        post = requests.post(f"{self.api_url}/projects/{self.project}/issues",
+        post = requests.post(self.url,
                           params  = payload,
                           headers = self.auth_headers,
                           timeout = TIMEOUT)
         return IssuePoster.check_req(post, name)
 
+    def search_params(self, expected_title: str) -> bool:
+        return { 'per_page': 1, 'search': expected_title, 'sort': 'asc', 'order_by': 'created_at' }
+
 
 class ForgejoPoster(IssuePoster):
     '''Forgejo (git.cgj.es instance) issue poster'''
@@ -101,6 +121,7 @@ class ForgejoPoster(IssuePoster):
         super().__init__(project, token, assignee, labels)
         self.auth_headers = { 'Authorization': f"token {self.token}" }
         self.api_url = f"{instance}/api/v1"
+        self.url = f"{instance}/api/v1/repos/{project}/issues"
 
     def post_issue(self, name: str, version: str, link: str) -> bool:
         payload = {
@@ -109,12 +130,15 @@ class ForgejoPoster(IssuePoster):
                 "title": IssuePoster.title(name, version),
                 "labels": self.labels if self.labels else [],
         }
-        post = requests.post(f"{self.api_url}/repos/{self.project}/issues",
+        post = requests.post(self.url,
                           json    = payload,
                           headers = self.auth_headers,
                           timeout = TIMEOUT)
         return IssuePoster.check_req(post, name)
 
+    def search_params(self, expected_title: str) -> bool:
+        return { 'q': expected_title, 'state': 'all' }
+
     def list_labels(self) -> dict[int, str] | None:
         '''Lists the labels and their IDs for this repository.'''
         get = requests.get(f"{self.api_url}/repos/{self.project}/labels",
@@ -166,6 +190,7 @@ class FeedReader:
     def match_post_save(self, version: str, _id: str, link: str) -> bool:
         '''Checks if a version is new, posts an issue and saves it as such.
             If the version is not new, or the posting fails, the method stops and returns False.'''
+        # Match 1: with local file
         try:
             with open(CONFIG_DIR + self.name, encoding="utf-8") as file:
                 match = file.readline().strip("\n") == str(_id)
@@ -173,8 +198,12 @@ class FeedReader:
             match = False
         if match:
             return False
-        if not self.target.post_issue(self.name, version, link):
+        # Match 2: with repository issues
-            return False
+        if not self.target.issue_exists(self.name, version):
+            # Post the issue
+            if not self.target.post_issue(self.name, version, link):
+                return False
+        # Save to disk
         if not os.path.isdir(CONFIG_DIR):
             os.makedirs(CONFIG_DIR)
         with open(CONFIG_DIR + self.name, mode="w", encoding="utf-8") as file: