From cadbfdbb7b36b2da34abc720134490d7969c4fd4 Mon Sep 17 00:00:00 2001 From: Jean-Marie Favreau Date: Fri, 7 Feb 2025 10:25:14 +0100 Subject: [PATCH 1/6] Ajout d'informations si l'import plante --- .../custom_extractors/fbevents.py | 6 +- .../import_tasks/downloader.py | 21 +- .../locale/fr/LC_MESSAGES/django.po | 325 +++++++++--------- 3 files changed, 186 insertions(+), 166 deletions(-) diff --git a/src/agenda_culturel/import_tasks/custom_extractors/fbevents.py b/src/agenda_culturel/import_tasks/custom_extractors/fbevents.py index c47c8ba..ac413a2 100644 --- a/src/agenda_culturel/import_tasks/custom_extractors/fbevents.py +++ b/src/agenda_culturel/import_tasks/custom_extractors/fbevents.py @@ -5,6 +5,8 @@ from bs4 import BeautifulSoup import json import os from datetime import datetime +from django.utils.translation import gettext_lazy as _ + import logging @@ -93,6 +95,8 @@ class CExtractor(TwoStepsExtractor): self.add_event(default_values, **event) else: - logger.warning("cannot find any event in page") + raise Exception( + _("Cannot get Facebook event from {}").format(event_url) + ) diff --git a/src/agenda_culturel/import_tasks/downloader.py b/src/agenda_culturel/import_tasks/downloader.py index 905c130..7a9d1ba 100644 --- a/src/agenda_culturel/import_tasks/downloader.py +++ b/src/agenda_culturel/import_tasks/downloader.py @@ -66,9 +66,10 @@ class SimpleDownloader(Downloader): class ChromiumHeadlessDownloader(Downloader): - def __init__(self, pause=True, noimage=True): + def __init__(self, pause=True, noimage=True, proxy=False): super().__init__() self.pause = pause + self.proxy = proxy self.options = Options() self.options.add_argument("--headless=new") self.options.add_argument("--disable-dev-shm-usage") @@ -78,6 +79,10 @@ class ChromiumHeadlessDownloader(Downloader): self.options.add_argument("--disable-dev-shm-usage") self.options.add_argument("--disable-browser-side-navigation") self.options.add_argument("--disable-gpu") + if self.proxy: + self.options.add_argument("--proxy-server=socks5://127.0.0.1:12345") + + if noimage: self.options.add_experimental_option( "prefs", { @@ -117,24 +122,24 @@ class ChromiumHeadlessDownloader(Downloader): except StaleElementReferenceException as e: print(f">> {type(e).__name__}: {e.args}") - return None + raise Exception("Error during download: " + str(e)[:64] + '...') except NoSuchElementException as e: print(f">> {type(e).__name__}: {e.args}") - return None + raise Exception("Error during download: " + str(e)[:64] + '...') except TimeoutException as e: print(f">> {type(e).__name__}: {e.args}") - return None + raise Exception("Error during download: " + str(e)[:64] + '...') except WebDriverException as e: print(f">> {type(e).__name__}: {e.args}") - return None + raise Exception("Error during download: " + str(e)[:64] + '...') except SessionNotCreatedException as e: print(f">> {type(e).__name__}: {e.args}") - return None + raise Exception("Error during download: " + str(e)[:64] + '...') except Exception as e: print(f">> {type(e).__name__} line {e.__traceback__.tb_lineno} of {__file__}: {e.args}") - return None + raise Exception("Error during download: " + str(e)[:64] + '...') except: print(f">> General Exception: {URL}") - return None + raise Exception("Error during download: " + str(e)[:64] + '...') return doc diff --git a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po index 1911f98..aacc813 100644 --- a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po +++ b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: agenda_culturel\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2025-01-31 23:08+0100\n" +"POT-Creation-Date: 2025-02-07 10:17+0100\n" "PO-Revision-Date: 2023-10-29 14:16+0000\n" "Last-Translator: Jean-Marie Favreau \n" "Language-Team: Jean-Marie Favreau \n" @@ -110,12 +110,12 @@ msgstr "Non" msgid "Imported from" msgstr "Importé depuis" -#: agenda_culturel/filters.py:359 agenda_culturel/models.py:644 -#: agenda_culturel/models.py:2261 +#: agenda_culturel/filters.py:359 agenda_culturel/models.py:648 +#: agenda_culturel/models.py:2272 msgid "Status" msgstr "Status" -#: agenda_culturel/filters.py:360 agenda_culturel/models.py:2048 +#: agenda_culturel/filters.py:360 agenda_culturel/models.py:2058 msgid "Closed" msgstr "Fermé" @@ -124,7 +124,7 @@ msgid "Open" msgstr "Ouvert" #: agenda_culturel/filters.py:364 agenda_culturel/filters.py:365 -#: agenda_culturel/models.py:2042 +#: agenda_culturel/models.py:2052 msgid "Spam" msgstr "Spam" @@ -132,7 +132,7 @@ msgstr "Spam" msgid "Non spam" msgstr "Non spam" -#: agenda_culturel/filters.py:370 agenda_culturel/models.py:2063 +#: agenda_culturel/filters.py:370 agenda_culturel/models.py:2073 msgid "Type" msgstr "Type" @@ -163,11 +163,11 @@ msgstr "" msgid "Your email" msgstr "Votre adresse email" -#: agenda_culturel/forms.py:138 agenda_culturel/models.py:2032 +#: agenda_culturel/forms.py:138 agenda_culturel/models.py:2042 msgid "Your email address" msgstr "Votre adresse email" -#: agenda_culturel/forms.py:144 agenda_culturel/models.py:2055 +#: agenda_culturel/forms.py:144 agenda_culturel/models.py:2065 msgid "Comments" msgstr "Commentaires" @@ -183,19 +183,19 @@ msgid "Receive notification of publication or leave a message for moderation" msgstr "Être notifié de la publication ou laisser un message à la modération" #: agenda_culturel/forms.py:174 agenda_culturel/models.py:188 -#: agenda_culturel/models.py:649 agenda_culturel/models.py:2183 -#: agenda_culturel/models.py:2293 +#: agenda_culturel/models.py:653 agenda_culturel/models.py:2194 +#: agenda_culturel/models.py:2304 msgid "Category" msgstr "Catégorie" #: agenda_culturel/forms.py:180 agenda_culturel/forms.py:214 #: agenda_culturel/forms.py:245 agenda_culturel/forms.py:405 -#: agenda_culturel/models.py:229 agenda_culturel/models.py:756 +#: agenda_culturel/models.py:229 agenda_culturel/models.py:760 msgid "Tags" msgstr "Étiquettes" #: agenda_culturel/forms.py:191 agenda_culturel/forms.py:552 -#: agenda_culturel/models.py:844 +#: agenda_culturel/models.py:848 msgid "Event" msgstr "Événement" @@ -219,12 +219,12 @@ msgstr "Cet événement est récurrent" msgid "Details" msgstr "Détails" -#: agenda_culturel/forms.py:327 agenda_culturel/models.py:679 -#: agenda_culturel/models.py:2158 +#: agenda_culturel/forms.py:327 agenda_culturel/models.py:683 +#: agenda_culturel/models.py:2169 msgid "Location" msgstr "Localisation" -#: agenda_culturel/forms.py:331 agenda_culturel/models.py:712 +#: agenda_culturel/forms.py:331 agenda_culturel/models.py:716 msgid "Illustration" msgstr "Illustration" @@ -314,7 +314,7 @@ msgid "Apply category {} to the event {}" msgstr "Appliquer la catégorie {} à l'événement {}" #: agenda_culturel/forms.py:749 agenda_culturel/models.py:495 -#: agenda_culturel/models.py:2345 +#: agenda_culturel/models.py:2356 msgid "Place" msgstr "Lieu" @@ -352,14 +352,18 @@ msgstr "Informations" msgid "Add a comment" msgstr "Ajouter un commentaire" -#: agenda_culturel/import_tasks/extractor.py:189 +#: agenda_culturel/import_tasks/custom_extractors/fbevents.py:99 +msgid "Cannot get Facebook event from {}" +msgstr "Impossible de récupérer un événement Facebook depuis {}" + +#: agenda_culturel/import_tasks/extractor.py:202 msgid "Unknown title" msgstr "Titre inconnu" #: agenda_culturel/models.py:67 agenda_culturel/models.py:115 #: agenda_culturel/models.py:198 agenda_culturel/models.py:438 #: agenda_culturel/models.py:466 agenda_culturel/models.py:553 -#: agenda_culturel/models.py:2024 agenda_culturel/models.py:2112 +#: agenda_culturel/models.py:2034 agenda_culturel/models.py:2123 msgid "Name" msgstr "Nom" @@ -420,7 +424,7 @@ msgid "Tag name" msgstr "Nom de l'étiquette" #: agenda_culturel/models.py:203 agenda_culturel/models.py:478 -#: agenda_culturel/models.py:565 agenda_culturel/models.py:696 +#: agenda_culturel/models.py:565 agenda_culturel/models.py:700 msgid "Description" msgstr "Description" @@ -596,7 +600,7 @@ msgstr "Organisme" msgid "Organisations" msgstr "Organismes" -#: agenda_culturel/models.py:594 agenda_culturel/models.py:2153 +#: agenda_culturel/models.py:594 agenda_culturel/models.py:2164 msgid "Published" msgstr "Publié" @@ -612,47 +616,47 @@ msgstr "Corbeille" msgid "Author of the event creation" msgstr "Auteur de la création de l'événement" -#: agenda_culturel/models.py:613 +#: agenda_culturel/models.py:614 msgid "Author of the last importation" msgstr "Auteur de la dernière importation" -#: agenda_culturel/models.py:621 +#: agenda_culturel/models.py:623 msgid "Author of the last modification" msgstr "Auteur de la dernière modification" -#: agenda_culturel/models.py:629 +#: agenda_culturel/models.py:632 msgid "Author of the last moderation" msgstr "Auteur de la dernière modération" -#: agenda_culturel/models.py:640 +#: agenda_culturel/models.py:644 msgid "Title" msgstr "Titre" -#: agenda_culturel/models.py:656 +#: agenda_culturel/models.py:660 msgid "Start day" msgstr "Date de début" -#: agenda_culturel/models.py:659 +#: agenda_culturel/models.py:663 msgid "Start time" msgstr "Heure de début" -#: agenda_culturel/models.py:665 +#: agenda_culturel/models.py:669 msgid "End day" msgstr "Date de fin" -#: agenda_culturel/models.py:670 +#: agenda_culturel/models.py:674 msgid "End time" msgstr "Heure de fin" -#: agenda_culturel/models.py:674 +#: agenda_culturel/models.py:678 msgid "Recurrence" msgstr "Récurrence" -#: agenda_culturel/models.py:685 +#: agenda_culturel/models.py:689 msgid "Location (free form)" msgstr "Localisation (forme libre)" -#: agenda_culturel/models.py:687 +#: agenda_culturel/models.py:691 msgid "" "Address of the event in case its not available in the already known places " "(free form)" @@ -660,11 +664,11 @@ msgstr "" "Adresse d'un événement si elle n'est pas déjà présente dans la liste des " "lieux disponibles (forme libre)" -#: agenda_culturel/models.py:704 +#: agenda_culturel/models.py:708 msgid "Organisers" msgstr "Organisateurs" -#: agenda_culturel/models.py:706 +#: agenda_culturel/models.py:710 msgid "" "list of event organisers. Organizers will only be displayed if one of them " "does not normally use the venue." @@ -672,95 +676,99 @@ msgstr "" "Liste des organisateurs de l'événements. Les organisateurs seront affichés " "uniquement si au moins un d'entre eux n'utilise pas habituellement le lieu." -#: agenda_culturel/models.py:719 +#: agenda_culturel/models.py:723 msgid "Illustration (URL)" msgstr "Illustration (URL)" -#: agenda_culturel/models.py:720 +#: agenda_culturel/models.py:724 msgid "External URL of the illustration image" msgstr "URL externe de l'image illustrative" -#: agenda_culturel/models.py:726 +#: agenda_culturel/models.py:730 msgid "Illustration description" msgstr "Description de l'illustration" -#: agenda_culturel/models.py:727 +#: agenda_culturel/models.py:731 msgid "Alternative text used by screen readers for the image" msgstr "Texte alternatif utiliser par les lecteurs d'écrans pour l'image" -#: agenda_culturel/models.py:735 +#: agenda_culturel/models.py:739 msgid "Importation source" msgstr "Source d'importation" -#: agenda_culturel/models.py:736 +#: agenda_culturel/models.py:740 msgid "Importation source used to detect removed entries." msgstr "Source d'importation utilisée pour détecter les éléments supprimés/" -#: agenda_culturel/models.py:742 +#: agenda_culturel/models.py:746 msgid "UUIDs" msgstr "UUIDs" -#: agenda_culturel/models.py:743 +#: agenda_culturel/models.py:747 msgid "UUIDs from import to detect duplicated entries." msgstr "UUIDs utilisés pendant l'import pour détecter les entrées dupliquées" -#: agenda_culturel/models.py:749 +#: agenda_culturel/models.py:753 msgid "Online sources or ticketing" msgstr "Sources en ligne ou billetterie" -#: agenda_culturel/models.py:763 +#: agenda_culturel/models.py:767 msgid "Other versions" msgstr "" -#: agenda_culturel/models.py:845 +#: agenda_culturel/models.py:849 msgid "Events" msgstr "Événements" -#: agenda_culturel/models.py:1242 +#: agenda_culturel/models.py:1247 msgid "Your event has been published" msgstr "Ton événement a été publié" -#: agenda_culturel/models.py:1245 +#: agenda_culturel/models.py:1250 msgid "Your message has not been retained" msgstr "Ton événement n'a pas été retenu" -#: agenda_culturel/models.py:1402 -msgid "during import process" -msgstr "pendant le processus d'import" +#: agenda_culturel/models.py:1332 agenda_culturel/models.py:1996 +msgid "Warning" +msgstr "Warning" -#: agenda_culturel/models.py:1411 agenda_culturel/models.py:1417 -msgid "warning" -msgstr "attention" - -#: agenda_culturel/models.py:1413 +#: agenda_culturel/models.py:1332 agenda_culturel/models.py:1422 msgid "the date has not been imported correctly." msgstr "la date n'a pas été importée correctement." -#: agenda_culturel/models.py:1419 +#: agenda_culturel/models.py:1412 +msgid "during import process" +msgstr "pendant le processus d'import" + +#: agenda_culturel/models.py:1420 agenda_culturel/models.py:1426 +msgid "warning" +msgstr "attention" + +#: agenda_culturel/models.py:1428 msgid "the title has not been imported correctly." msgstr "le titre n'a pas été importé correctement." -#: agenda_culturel/models.py:1732 +#: agenda_culturel/models.py:1742 msgid "Updated field(s): " msgstr "Champ(s) mis à jour: " -#: agenda_culturel/models.py:1733 +#: agenda_culturel/models.py:1743 msgid "Update" msgstr "Mise à jour" -#: agenda_culturel/models.py:1734 +#: agenda_culturel/models.py:1744 msgid "update process" msgstr "processus de mise à jour" -#: agenda_culturel/models.py:1778 +#: agenda_culturel/models.py:1788 msgid "Import" msgstr "Import" -#: agenda_culturel/models.py:1779 +#: agenda_culturel/models.py:1789 msgid "import process" msgstr "processus d'import" -#: agenda_culturel/models.py:1780 +#: agenda_culturel/models.py:1790 msgid "" "The duration of the event is a little too long for direct publication. " "Moderators can choose to publish it or not." @@ -768,175 +776,175 @@ msgstr "" "La durée de l'événement est un peu trop longue pour qu'il soit publié " "directement. Les modérateurs peuvent choisir de le publier ou non." -#: agenda_culturel/models.py:1980 +#: agenda_culturel/models.py:1990 msgid "From contributor" msgstr "D'un·e contributeurice" -#: agenda_culturel/models.py:1981 +#: agenda_culturel/models.py:1991 msgid "Import process" msgstr "Processus d'import" -#: agenda_culturel/models.py:1982 +#: agenda_culturel/models.py:1992 msgid "Update process" msgstr "Processus de mise à jour" -#: agenda_culturel/models.py:1983 +#: agenda_culturel/models.py:1993 msgid "Contact form" msgstr "Formulaire de contact" -#: agenda_culturel/models.py:1984 +#: agenda_culturel/models.py:1994 msgid "Event report" msgstr "Signalemet d'événement" -#: agenda_culturel/models.py:1985 +#: agenda_culturel/models.py:1995 msgid "From contributor (without message)" msgstr "D'un·e contributeurice (sans message)" -#: agenda_culturel/models.py:1986 -msgid "Warning" -msgstr "Warning" - -#: agenda_culturel/models.py:1989 agenda_culturel/models.py:2037 +#: agenda_culturel/models.py:1999 agenda_culturel/models.py:2047 msgid "Message" msgstr "Message" -#: agenda_culturel/models.py:1990 +#: agenda_culturel/models.py:2000 msgid "Messages" msgstr "Messages" -#: agenda_culturel/models.py:2001 +#: agenda_culturel/models.py:2011 msgid "Subject" msgstr "Sujet" -#: agenda_culturel/models.py:2002 +#: agenda_culturel/models.py:2012 msgid "The subject of your message" msgstr "Sujet de votre message" -#: agenda_culturel/models.py:2008 +#: agenda_culturel/models.py:2018 msgid "Related event" msgstr "Événement associé" -#: agenda_culturel/models.py:2009 +#: agenda_culturel/models.py:2019 msgid "The message is associated with this event." msgstr "Le message est associé à cet événement." -#: agenda_culturel/models.py:2017 +#: agenda_culturel/models.py:2027 msgid "Author of the message" msgstr "Auteur du message" -#: agenda_culturel/models.py:2025 +#: agenda_culturel/models.py:2035 msgid "Your name" msgstr "Votre nom" -#: agenda_culturel/models.py:2031 +#: agenda_culturel/models.py:2041 msgid "Email address" msgstr "Adresse email" -#: agenda_culturel/models.py:2037 +#: agenda_culturel/models.py:2047 msgid "Your message" msgstr "Votre message" -#: agenda_culturel/models.py:2043 +#: agenda_culturel/models.py:2053 msgid "This message is a spam." msgstr "Ce message est un spam." -#: agenda_culturel/models.py:2050 +#: agenda_culturel/models.py:2060 msgid "this message has been processed and no longer needs to be handled" msgstr "Ce message a été traité et ne nécessite plus d'être pris en charge" -#: agenda_culturel/models.py:2056 +#: agenda_culturel/models.py:2066 msgid "Comments on the message from the moderation team" msgstr "Commentaires sur ce message par l'équipe de modération" -#: agenda_culturel/models.py:2078 agenda_culturel/models.py:2241 +#: agenda_culturel/models.py:2088 agenda_culturel/models.py:2252 msgid "Recurrent import" msgstr "Import récurrent" -#: agenda_culturel/models.py:2079 +#: agenda_culturel/models.py:2089 msgid "Recurrent imports" msgstr "Imports récurrents" -#: agenda_culturel/models.py:2083 +#: agenda_culturel/models.py:2093 msgid "ical" msgstr "ical" -#: agenda_culturel/models.py:2084 +#: agenda_culturel/models.py:2094 msgid "ical no busy" msgstr "ical sans busy" -#: agenda_culturel/models.py:2085 +#: agenda_culturel/models.py:2095 msgid "ical no VC" msgstr "ical sans VC" -#: agenda_culturel/models.py:2086 +#: agenda_culturel/models.py:2096 msgid "lacoope.org" msgstr "lacoope.org" -#: agenda_culturel/models.py:2087 +#: agenda_culturel/models.py:2097 msgid "la comédie" msgstr "la comédie" -#: agenda_culturel/models.py:2088 +#: agenda_culturel/models.py:2098 msgid "le fotomat" msgstr "le fotomat" -#: agenda_culturel/models.py:2089 +#: agenda_culturel/models.py:2099 msgid "la puce à l'oreille" msgstr "la puce à loreille" -#: agenda_culturel/models.py:2090 +#: agenda_culturel/models.py:2100 msgid "Plugin wordpress MEC" msgstr "Plugin wordpress MEC" -#: agenda_culturel/models.py:2091 +#: agenda_culturel/models.py:2101 msgid "Événements d'une page FB" msgstr "Événements d'une page FB" -#: agenda_culturel/models.py:2092 -msgid "la cour des 3 coquins" -msgstr "la cour des 3 coquins" +#: agenda_culturel/models.py:2102 +msgid "Billetterie Clermont-Ferrand" +msgstr "" -#: agenda_culturel/models.py:2093 +#: agenda_culturel/models.py:2103 msgid "Arachnée concert" msgstr "Arachnée concert" -#: agenda_culturel/models.py:2094 +#: agenda_culturel/models.py:2104 msgid "Le Rio" msgstr "Le Rio" -#: agenda_culturel/models.py:2095 +#: agenda_culturel/models.py:2105 msgid "La Raymonde" msgstr "La Raymone" -#: agenda_culturel/models.py:2096 +#: agenda_culturel/models.py:2106 msgid "Agenda apidae tourisme" msgstr "Agenda apidae tourisme" -#: agenda_culturel/models.py:2097 +#: agenda_culturel/models.py:2107 msgid "Agenda iguana (médiathèques)" msgstr "Agenda iguana (médiathèques)" -#: agenda_culturel/models.py:2100 +#: agenda_culturel/models.py:2108 +msgid "Mille formes" +msgstr "" + +#: agenda_culturel/models.py:2111 msgid "simple" msgstr "simple" -#: agenda_culturel/models.py:2101 +#: agenda_culturel/models.py:2112 msgid "Headless Chromium" msgstr "chromium sans interface" -#: agenda_culturel/models.py:2102 +#: agenda_culturel/models.py:2113 msgid "Headless Chromium (pause)" msgstr "chromium sans interface (pause)" -#: agenda_culturel/models.py:2107 +#: agenda_culturel/models.py:2118 msgid "daily" msgstr "chaque jour" -#: agenda_culturel/models.py:2109 +#: agenda_culturel/models.py:2120 msgid "weekly" msgstr "chaque semaine" -#: agenda_culturel/models.py:2114 +#: agenda_culturel/models.py:2125 msgid "" "Recurrent import name. Be careful to choose a name that is easy to " "understand, as it will be public and displayed on the sites About page." @@ -944,151 +952,151 @@ msgstr "" "Nom de l'import récurrent. Attention à choisir un nom compréhensible, car il " "sera public, et affiché sur la page à propos du site." -#: agenda_culturel/models.py:2121 +#: agenda_culturel/models.py:2132 msgid "Processor" msgstr "Processeur" -#: agenda_culturel/models.py:2124 +#: agenda_culturel/models.py:2135 msgid "Downloader" msgstr "Téléchargeur" -#: agenda_culturel/models.py:2131 +#: agenda_culturel/models.py:2142 msgid "Import recurrence" msgstr "Récurrence d'import" -#: agenda_culturel/models.py:2138 +#: agenda_culturel/models.py:2149 msgid "Source" msgstr "Source" -#: agenda_culturel/models.py:2139 +#: agenda_culturel/models.py:2150 msgid "URL of the source document" msgstr "URL du document source" -#: agenda_culturel/models.py:2143 +#: agenda_culturel/models.py:2154 msgid "Browsable url" msgstr "URL navigable" -#: agenda_culturel/models.py:2145 +#: agenda_culturel/models.py:2156 msgid "URL of the corresponding document that will be shown to visitors." msgstr "URL correspondant au document et qui sera montrée aux visiteurs" -#: agenda_culturel/models.py:2154 +#: agenda_culturel/models.py:2165 msgid "Status of each imported event (published or draft)" msgstr "Status de chaque événement importé (publié ou brouillon)" -#: agenda_culturel/models.py:2159 +#: agenda_culturel/models.py:2170 msgid "Address for each imported event" msgstr "Adresse de chaque événement importé" -#: agenda_culturel/models.py:2166 +#: agenda_culturel/models.py:2177 msgid "Force location" msgstr "Focer la localisation" -#: agenda_culturel/models.py:2167 +#: agenda_culturel/models.py:2178 msgid "force location even if another is detected." msgstr "Forcer la localisation même si une autre a été détectée." -#: agenda_culturel/models.py:2173 +#: agenda_culturel/models.py:2184 msgid "Organiser" msgstr "Organisateur" -#: agenda_culturel/models.py:2174 +#: agenda_culturel/models.py:2185 msgid "Organiser of each imported event" msgstr "Organisateur de chaque événement importé" -#: agenda_culturel/models.py:2184 +#: agenda_culturel/models.py:2195 msgid "Category of each imported event" msgstr "Catégorie de chaque événement importé" -#: agenda_culturel/models.py:2192 +#: agenda_culturel/models.py:2203 msgid "Tags for each imported event" msgstr "Étiquettes de chaque événement importé" -#: agenda_culturel/models.py:2193 +#: agenda_culturel/models.py:2204 msgid "A list of tags that describe each imported event." msgstr "Une liste d'étiquettes décrivant chaque événement importé" -#: agenda_culturel/models.py:2222 +#: agenda_culturel/models.py:2233 msgid "Running" msgstr "En cours" -#: agenda_culturel/models.py:2223 +#: agenda_culturel/models.py:2234 msgid "Canceled" msgstr "Annulé" -#: agenda_culturel/models.py:2224 +#: agenda_culturel/models.py:2235 msgid "Success" msgstr "Succès" -#: agenda_culturel/models.py:2225 +#: agenda_culturel/models.py:2236 msgid "Failed" msgstr "Erreur" -#: agenda_culturel/models.py:2228 +#: agenda_culturel/models.py:2239 msgid "Batch importation" msgstr "Importation par lot" -#: agenda_culturel/models.py:2229 +#: agenda_culturel/models.py:2240 msgid "Batch importations" msgstr "Importations par lot" -#: agenda_culturel/models.py:2242 +#: agenda_culturel/models.py:2253 msgid "Reference to the recurrent import processing" msgstr "Référence du processus d'import récurrent" -#: agenda_culturel/models.py:2250 +#: agenda_culturel/models.py:2261 msgid "URL (if not recurrent import)" msgstr "URL (si pas d'import récurrent)" -#: agenda_culturel/models.py:2252 +#: agenda_culturel/models.py:2263 msgid "Source URL if no RecurrentImport is associated." msgstr "URL source si aucun import récurrent n'est associé" -#: agenda_culturel/models.py:2265 +#: agenda_culturel/models.py:2276 msgid "Error message" msgstr "Votre message" -#: agenda_culturel/models.py:2269 +#: agenda_culturel/models.py:2280 msgid "Number of collected events" msgstr "Nombre d'événements collectés" -#: agenda_culturel/models.py:2272 +#: agenda_culturel/models.py:2283 msgid "Number of imported events" msgstr "Nombre d'événements importés" -#: agenda_culturel/models.py:2275 +#: agenda_culturel/models.py:2286 msgid "Number of updated events" msgstr "Nombre d'événements mis à jour" -#: agenda_culturel/models.py:2278 +#: agenda_culturel/models.py:2289 msgid "Number of removed events" msgstr "Nombre d'événements supprimés" -#: agenda_culturel/models.py:2286 +#: agenda_culturel/models.py:2297 msgid "Weight" msgstr "Poids" -#: agenda_culturel/models.py:2287 +#: agenda_culturel/models.py:2298 msgid "The lower is the weight, the earlier the filter is applied" msgstr "Plus le poids est léger, plus le filtre sera appliqué tôt" -#: agenda_culturel/models.py:2294 +#: agenda_culturel/models.py:2305 msgid "Category applied to the event" msgstr "Catégorie appliquée à l'événement" -#: agenda_culturel/models.py:2299 +#: agenda_culturel/models.py:2310 msgid "Contained in the title" msgstr "Contenu dans le titre" -#: agenda_culturel/models.py:2300 +#: agenda_culturel/models.py:2311 msgid "Text contained in the event title" msgstr "Texte contenu dans le titre de l'événement" -#: agenda_culturel/models.py:2306 +#: agenda_culturel/models.py:2317 msgid "Exact title extract" msgstr "Extrait exact du titre" -#: agenda_culturel/models.py:2308 +#: agenda_culturel/models.py:2319 msgid "" "If checked, the extract will be searched for in the title using the exact " "form (capitals, accents)." @@ -1096,19 +1104,19 @@ msgstr "" "Si coché, l'extrait sera recherché dans le titre en utilisant la forme " "exacte (majuscules, accents)" -#: agenda_culturel/models.py:2314 +#: agenda_culturel/models.py:2325 msgid "Contained in the description" msgstr "Contenu dans la description" -#: agenda_culturel/models.py:2315 +#: agenda_culturel/models.py:2326 msgid "Text contained in the description" msgstr "Texte contenu dans la description" -#: agenda_culturel/models.py:2321 +#: agenda_culturel/models.py:2332 msgid "Exact description extract" msgstr "Extrait exact de description" -#: agenda_culturel/models.py:2323 +#: agenda_culturel/models.py:2334 msgid "" "If checked, the extract will be searched for in the description using the " "exact form (capitals, accents)." @@ -1116,19 +1124,19 @@ msgstr "" "Si coché, l'extrait sera recherché dans la description en utilisant la forme " "exacte (majuscules, accents)" -#: agenda_culturel/models.py:2329 +#: agenda_culturel/models.py:2340 msgid "Contained in the location" msgstr "Contenu dans la localisation" -#: agenda_culturel/models.py:2330 +#: agenda_culturel/models.py:2341 msgid "Text contained in the event location" msgstr "Texte contenu dans la localisation de l'événement" -#: agenda_culturel/models.py:2336 +#: agenda_culturel/models.py:2347 msgid "Exact location extract" msgstr "Extrait exact de localisation" -#: agenda_culturel/models.py:2338 +#: agenda_culturel/models.py:2349 msgid "" "If checked, the extract will be searched for in the location using the exact " "form (capitals, accents)." @@ -1136,15 +1144,15 @@ msgstr "" "Si coché, l'extrait sera recherché dans la localisation en utilisant la " "forme exacte (majuscules, accents)" -#: agenda_culturel/models.py:2346 +#: agenda_culturel/models.py:2357 msgid "Location from place" msgstr "Localisation depuis le lieu" -#: agenda_culturel/models.py:2355 +#: agenda_culturel/models.py:2366 msgid "Categorisation rule" msgstr "Règle de catégorisation" -#: agenda_culturel/models.py:2356 +#: agenda_culturel/models.py:2367 msgid "Categorisation rules" msgstr "Règles de catégorisation" @@ -1486,3 +1494,6 @@ msgstr "L'événement {} a été supprimé avec succès." #: agenda_culturel/views.py:2343 msgid "Cache successfully cleared." msgstr "Le cache a été vidé avec succès." + +#~ msgid "la cour des 3 coquins" +#~ msgstr "la cour des 3 coquins" From 115644c1122a693905596469e58c74f796ff4948 Mon Sep 17 00:00:00 2001 From: Jean-Marie Favreau Date: Fri, 7 Feb 2025 10:29:30 +0100 Subject: [PATCH 2/6] Ajout d'une doc sur le proxy socket --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 98ae204..12f7a89 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,15 @@ On peut aussi peupler les positions de référence qui serviront aux recherches * ```make create-reference-locations``` +## Utilisation d'un proxy socket +On peut activer à la main (pour l'instant) un proxy type socket pour l'import d'événements. + +* se connecter au docker du celery worker : ```docker exec -it agenda_culturel-celery-worker bash``` +* mettre à jour les dépôts ```apt update``` +* installer le client ssh ```apt install ssh-client``` +* créer un socket ssh ```sh -D 12345 USER@HOST``` +* modifier le drapeau proxy dans le constructeur de [downloader.py](src/agenda_culturel/import_tasks/downloader.py). ## Notes aux développeurs From bb2a6b04e50b7093b66866a25f28044c5d30c567 Mon Sep 17 00:00:00 2001 From: Jean-Marie Favreau Date: Fri, 7 Feb 2025 11:02:33 +0100 Subject: [PATCH 3/6] On alerte que l'heure manque dans les imports de la raymonde --- .../import_tasks/custom_extractors/laraymonde.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/agenda_culturel/import_tasks/custom_extractors/laraymonde.py b/src/agenda_culturel/import_tasks/custom_extractors/laraymonde.py index 4249bba..ced6100 100644 --- a/src/agenda_culturel/import_tasks/custom_extractors/laraymonde.py +++ b/src/agenda_culturel/import_tasks/custom_extractors/laraymonde.py @@ -32,20 +32,22 @@ class CExtractor(TwoStepsExtractorNoPause): title = soup.select_one(".showDesc h4 a.summary").text start_day = soup.select_one(".showDate .value-title") + start_time = None if not start_day is None: start_day = start_day["title"] if not start_day is None: start_day = start_day.split("T")[0] - if start_day is None: - print("impossible de récupérer la date") - return + description = soup.select_one('.showDetails.description').text image = soup.select('.showDetails.description img') if not image is None: image_alt = image[-1]["alt"] image = image[-1]["src"] + if start_time is None: + title += " - Attention: la date n'a pu être extraite" + self.add_event_with_props( default_values, event_url, @@ -58,7 +60,7 @@ class CExtractor(TwoStepsExtractorNoPause): recurrences=None, uuids=[event_url], url_human=event_url, - start_time=None, + start_time=start_time, end_day=None, end_time=None, published=published, From 7648d1d79410aa284fe0a90cd791d3ba1f5ea7a4 Mon Sep 17 00:00:00 2001 From: Jean-Marie Favreau Date: Fri, 7 Feb 2025 11:55:32 +0100 Subject: [PATCH 4/6] =?UTF-8?q?Ajout=20d'un=20d=C3=A9tecteur=20de=20pages?= =?UTF-8?q?=20pas=20compl=C3=A8tement=20charg=C3=A9es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../custom_extractors/fbevents.py | 30 ++++++++++--------- .../locale/fr/LC_MESSAGES/django.po | 14 +++++++-- 2 files changed, 27 insertions(+), 17 deletions(-) diff --git a/src/agenda_culturel/import_tasks/custom_extractors/fbevents.py b/src/agenda_culturel/import_tasks/custom_extractors/fbevents.py index ac413a2..49b72c8 100644 --- a/src/agenda_culturel/import_tasks/custom_extractors/fbevents.py +++ b/src/agenda_culturel/import_tasks/custom_extractors/fbevents.py @@ -17,30 +17,28 @@ logger = logging.getLogger(__name__) # such as https://www.facebook.com/laJeteeClermont/events class CExtractor(TwoStepsExtractor): - def find_event_id_fragment_in_array(self, array, first=True): - found = False + def find_event_id_fragment_in_array(self, array): if isinstance(array, dict): if "__typename" in array and array["__typename"] == "Event" and "id" in array: self.add_event_url("https://www.facebook.com/events/" + array["id"] + "/") - found = True - if not found: + self.found = True + else: for k in array: - found = self.find_event_id_fragment_in_array(array[k], False) or found + if k == "pageItems": + self.has_page_items = True + self.find_event_id_fragment_in_array(array[k]) elif isinstance(array, list): for e in array: - found = self.find_event_id_fragment_in_array(e, False) or found - return found + self.find_event_id_fragment_in_array(e) def find_in_js(self, soup): - found = False for json_script in soup.find_all("script", type="application/json"): json_txt = json_script.get_text() json_struct = json.loads(json_txt) - found = self.find_event_id_fragment_in_array(json_struct) or found + self.find_event_id_fragment_in_array(json_struct) - return found def build_event_url_list(self, content): @@ -48,16 +46,20 @@ class CExtractor(TwoStepsExtractor): debug = False - found = False + self.found = False links = soup.find_all("a") for link in links: if link.get("href").startswith('https://www.facebook.com/events/'): self.add_event_url(link.get('href').split('?')[0]) - found = True + self.found = True - found = self.find_in_js(soup) or found + self.has_page_items = False + self.find_in_js(soup) - if not found and debug: + if not self.has_page_items: + raise Exception(_("the page was not yet populated with events, so the loading time was probably too short")) + + if not self.found and debug: directory = "errors/" if not os.path.exists(directory): os.makedirs(directory) diff --git a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po index aacc813..415d93f 100644 --- a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po +++ b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po @@ -2,13 +2,13 @@ # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER # This file is distributed under the same license as the PACKAGE package. # Jean-Marie Favreau , 2023. -# +# #, fuzzy msgid "" msgstr "" "Project-Id-Version: agenda_culturel\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2025-02-07 10:17+0100\n" +"POT-Creation-Date: 2025-02-07 11:55+0100\n" "PO-Revision-Date: 2023-10-29 14:16+0000\n" "Last-Translator: Jean-Marie Favreau \n" "Language-Team: Jean-Marie Favreau \n" @@ -352,7 +352,15 @@ msgstr "Informations" msgid "Add a comment" msgstr "Ajouter un commentaire" -#: agenda_culturel/import_tasks/custom_extractors/fbevents.py:99 +#: agenda_culturel/import_tasks/custom_extractors/fbevents.py:60 +msgid "" +"the page was not yet populated with events, so the loading time was probably " +"too short" +msgstr "" +"la page n'était pas encore peuplée des événements, le temps de chargement a " +"sans doute été trop court" + +#: agenda_culturel/import_tasks/custom_extractors/fbevents.py:101 msgid "Cannot get Facebook event from {}" msgstr "Impossible de récupérer un événement Facebook depuis {}" From ce7576e9ae5ba63fb7eeff7537896d7da759880f Mon Sep 17 00:00:00 2001 From: Jean-Marie Favreau Date: Fri, 7 Feb 2025 16:03:04 +0100 Subject: [PATCH 5/6] =?UTF-8?q?On=20visualise=20les=20=C3=A9v=C3=A9nements?= =?UTF-8?q?=20import=C3=A9s=20mais=20hors=20imports=20r=C3=A9currents?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/agenda_culturel/filters.py | 42 ++++++++++++++++++- .../locale/fr/LC_MESSAGES/django.po | 34 +++++++++------ src/agenda_culturel/static/style.scss | 11 ++++- 3 files changed, 72 insertions(+), 15 deletions(-) diff --git a/src/agenda_culturel/filters.py b/src/agenda_culturel/filters.py index b7d139c..712dcfe 100644 --- a/src/agenda_culturel/filters.py +++ b/src/agenda_culturel/filters.py @@ -2,7 +2,7 @@ import django_filters from django.utils.translation import gettext_lazy as _ from django import forms from django.contrib.postgres.search import SearchQuery, SearchHeadline -from django.db.models import Count, Q +from django.db.models import Count, Q, F from datetime import date, timedelta from urllib.parse import urlparse, parse_qs, urlencode @@ -326,6 +326,18 @@ class EventFilterAdmin(django_filters.FilterSet): method="filter_by_representative", widget=forms.CheckboxSelectMultiple) + pure_import = django_filters.MultipleChoiceFilter( + label=_("Pure import"), + choices=[(True, _("Yes")), (False, _("No"))], + method="filter_by_pure_import", + widget=forms.CheckboxSelectMultiple) + + in_recurrent_import = django_filters.MultipleChoiceFilter( + label=_("In recurrent import"), + choices=[(True, _("Yes")), (False, _("No"))], + method="filter_by_in_recurrent_import", + widget=forms.CheckboxSelectMultiple) + import_sources = django_filters.ModelChoiceFilter( label=_("Imported from"), method="filter_by_source", @@ -336,6 +348,32 @@ class EventFilterAdmin(django_filters.FilterSet): src = RecurrentImport.objects.get(pk=value.pk).source return queryset.filter(import_sources__contains=[src]) + def filter_by_in_recurrent_import(self, queryset, name, value): + if value is None or len(value) != 1: + return queryset + else: + srcs = RecurrentImport.objects.all().values_list("source") + q = Q(import_sources__overlap=srcs) + if value[0] == 'True': + print(q) + return queryset.filter(q) + else: + return queryset.exclude(q) + + def filter_by_pure_import(self, queryset, name, value): + if value is None or len(value) != 1: + return queryset + else: + q = (Q(import_sources__isnull=False) & + (Q(modified_date__isnull=True) | + Q(modified_date__lte=F('imported_date')))) + if value[0] == 'True': + print(q) + return queryset.filter(q) + else: + return queryset.exclude(q) + + def filter_by_representative(self, queryset, name, value): if value is None or len(value) != 1: return queryset @@ -343,7 +381,7 @@ class EventFilterAdmin(django_filters.FilterSet): q = (Q(other_versions__isnull=True) | Q(other_versions__representative=F('pk')) | Q(other_versions__representative__isnull=True)) - if value[0] == True: + if value[0] == 'True': return queryset.filter(q) else: return queryset.exclude(q) diff --git a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po index 415d93f..2812590 100644 --- a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po +++ b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po @@ -2,13 +2,13 @@ # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER # This file is distributed under the same license as the PACKAGE package. # Jean-Marie Favreau , 2023. -# +# #, fuzzy msgid "" msgstr "" "Project-Id-Version: agenda_culturel\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2025-02-07 11:55+0100\n" +"POT-Creation-Date: 2025-02-07 16:00+0100\n" "PO-Revision-Date: 2023-10-29 14:16+0000\n" "Last-Translator: Jean-Marie Favreau \n" "Language-Team: Jean-Marie Favreau \n" @@ -98,49 +98,59 @@ msgstr "Choisir une localité" msgid "Representative version" msgstr "Version représentative" -#: agenda_culturel/filters.py:325 agenda_culturel/filters.py:395 +#: agenda_culturel/filters.py:325 agenda_culturel/filters.py:331 +#: agenda_culturel/filters.py:337 agenda_culturel/filters.py:433 msgid "Yes" msgstr "Oui" -#: agenda_culturel/filters.py:325 agenda_culturel/filters.py:395 +#: agenda_culturel/filters.py:325 agenda_culturel/filters.py:331 +#: agenda_culturel/filters.py:337 agenda_culturel/filters.py:433 msgid "No" msgstr "Non" #: agenda_culturel/filters.py:330 +msgid "Pure import" +msgstr "Événement importé" + +#: agenda_culturel/filters.py:336 +msgid "In recurrent import" +msgstr "Inclut dans un import récurrent" + +#: agenda_culturel/filters.py:342 msgid "Imported from" msgstr "Importé depuis" -#: agenda_culturel/filters.py:359 agenda_culturel/models.py:648 +#: agenda_culturel/filters.py:397 agenda_culturel/models.py:648 #: agenda_culturel/models.py:2272 msgid "Status" msgstr "Status" -#: agenda_culturel/filters.py:360 agenda_culturel/models.py:2058 +#: agenda_culturel/filters.py:398 agenda_culturel/models.py:2058 msgid "Closed" msgstr "Fermé" -#: agenda_culturel/filters.py:360 +#: agenda_culturel/filters.py:398 msgid "Open" msgstr "Ouvert" -#: agenda_culturel/filters.py:364 agenda_culturel/filters.py:365 +#: agenda_culturel/filters.py:402 agenda_culturel/filters.py:403 #: agenda_culturel/models.py:2052 msgid "Spam" msgstr "Spam" -#: agenda_culturel/filters.py:365 +#: agenda_culturel/filters.py:403 msgid "Non spam" msgstr "Non spam" -#: agenda_culturel/filters.py:370 agenda_culturel/models.py:2073 +#: agenda_culturel/filters.py:408 agenda_culturel/models.py:2073 msgid "Type" msgstr "Type" -#: agenda_culturel/filters.py:382 +#: agenda_culturel/filters.py:420 msgid "Search" msgstr "Rechercher" -#: agenda_culturel/filters.py:394 +#: agenda_culturel/filters.py:432 msgid "In the past" msgstr "Dans le passé" diff --git a/src/agenda_culturel/static/style.scss b/src/agenda_culturel/static/style.scss index b589711..aa1d8ca 100644 --- a/src/agenda_culturel/static/style.scss +++ b/src/agenda_culturel/static/style.scss @@ -620,11 +620,20 @@ header .remarque { } .form.recent, .form.main-filter, .search .form { - #id_status>div, #id_representative>div { + #id_status>div, #id_representative>div, #id_pure_import>div, #id_in_recurrent_import>div { display: inline-block; margin-right: 2em; } } +.form.recent { + display: grid; + grid-template-columns: repeat(2, 1fr); + column-gap: .5em; + :last-child, :nth-last-child(4), + button { + grid-column: 1/3; + } +} #search { form { From c7958030733a998dce53840aa3e3c2ee81a1f3e8 Mon Sep 17 00:00:00 2001 From: Jean-Marie Favreau Date: Fri, 7 Feb 2025 16:44:45 +0100 Subject: [PATCH 6/6] =?UTF-8?q?Ajout=20d'informations=20sur=20les=20=C3=A9?= =?UTF-8?q?v=C3=A9nements=20import=C3=A9s=20=C3=A0=20venir?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../templates/agenda_culturel/administration.html | 10 ++++++++-- src/agenda_culturel/views.py | 13 ++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/src/agenda_culturel/templates/agenda_culturel/administration.html b/src/agenda_culturel/templates/agenda_culturel/administration.html index ab9325b..d851f3e 100644 --- a/src/agenda_culturel/templates/agenda_culturel/administration.html +++ b/src/agenda_culturel/templates/agenda_culturel/administration.html @@ -60,8 +60,7 @@ {% endfor %} - - +

Activité des derniers jours

@@ -75,6 +74,13 @@

Détail des imports récurrents : {% include "agenda_culturel/rimports-info-inc.html" with all=1 %}

+

Synthèse des événements importés

+

En ne considérant que les événements à venir, on compte :

+
    +
  • Nombre d'événements issus d'importations récurrentes : {{ nb_in_rimport }}
  • +
  • Nombre d'événements issus d'importations uniques : {{ nb_in_orphan_import }}
  • +
+
diff --git a/src/agenda_culturel/views.py b/src/agenda_culturel/views.py index 6cd71fb..f6d2c74 100644 --- a/src/agenda_culturel/views.py +++ b/src/agenda_culturel/views.py @@ -1037,6 +1037,16 @@ def administration(request): .count()) nb_all = imported_events.count() + # get some info about imported (or not) events + srcs = RecurrentImport.objects.all().values_list("source") + in_future = Event.objects.filter(Q(start_day__gte=today)) + nb_in_rimport = in_future.filter(Q(import_sources__overlap=srcs)).count() + nb_in_orphan_import = in_future.filter( + (Q(import_sources__isnull=False) & + (Q(modified_date__isnull=True) | + Q(modified_date__lte=F('imported_date')))) + & ~Q(import_sources__overlap=srcs)).count() + # get all non moderated events nb_not_moderated = Event.get_nb_not_moderated(today, nb_mod_days, nb_classes) @@ -1048,7 +1058,8 @@ def administration(request): "events": events, "batch_imports": batch_imports, "nb_failed": nb_failed, "nb_canceled": nb_canceled, "nb_running": nb_running, "nb_all": nb_all, - "nb_not_moderated": nb_not_moderated}, + "nb_not_moderated": nb_not_moderated, + "nb_in_rimport": nb_in_rimport, "nb_in_orphan_import": nb_in_orphan_import}, )