From fc579998ca5bd74e50f067e4d7a456d264b298cc Mon Sep 17 00:00:00 2001 From: Jean-Marie Favreau Date: Sun, 9 Feb 2025 14:42:15 +0100 Subject: [PATCH] =?UTF-8?q?Ajout=20d'un=20import=20des=20=C3=A9v=C3=A9neme?= =?UTF-8?q?nts=20orphelins?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/agenda_culturel/celery.py | 40 +- src/agenda_culturel/import_tasks/extractor.py | 6 +- .../import_tasks/extractor_facebook.py | 9 +- .../locale/fr/LC_MESSAGES/django.po | 391 +++++++++--------- src/agenda_culturel/models.py | 62 ++- .../agenda_culturel/batch-imports-inc.html | 3 + .../templates/agenda_culturel/imports.html | 8 +- .../run_orphan_imports_confirm.html | 27 ++ src/agenda_culturel/urls.py | 2 + src/agenda_culturel/views.py | 59 ++- 10 files changed, 383 insertions(+), 224 deletions(-) create mode 100644 src/agenda_culturel/templates/agenda_culturel/run_orphan_imports_confirm.html diff --git a/src/agenda_culturel/celery.py b/src/agenda_culturel/celery.py index cf84220..2be98a3 100644 --- a/src/agenda_culturel/celery.py +++ b/src/agenda_culturel/celery.py @@ -324,11 +324,18 @@ def weekly_imports(self): run_recurrent_imports_from_list([imp.pk for imp in imports]) @app.task(base=ChromiumTask, bind=True) -def import_events_from_url(self, url, cat, tags, force=False, user_id=None, email=None, comments=None): +def import_events_from_url(self, urls, cat=None, tags=None, force=False, user_id=None, email=None, comments=None): from .db_importer import DBImporterEvents from agenda_culturel.models import RecurrentImport, BatchImportation from agenda_culturel.models import Event, Category + if isinstance(urls, list): + url = urls[0] + is_list = True + else: + is_list = False + url = urls + with memcache_chromium_lock(self.app.oid) as acquired: if acquired: @@ -386,7 +393,7 @@ def import_events_from_url(self, url, cat, tags, force=False, user_id=None, emai logger.error(e) close_import_task(self.request.id, False, e, importer) - return + return urls[1:] if is_list else True # if chromium is locked, we wait 30 seconds before retrying raise self.retry(countdown=30) @@ -403,7 +410,36 @@ def import_events_from_urls(self, urls_cat_tags, user_id=None, email=None, comme import_events_from_url.delay(url, cat, tags, user_id=user_id, email=email, comments=comments) +@app.task(base=ChromiumTask, bind=True) +def update_orphan_pure_import_events(self): + from agenda_culturel.models import RecurrentImport + from agenda_culturel.models import Event + from django.db.models import Q, F + + # get all recurrent sources + srcs = RecurrentImport.objects.all().values_list("source") + today = date.today() + # get all events in future with a source and not related to a recurrent import + urls = Event.objects.filter(Q(start_day__gte=today)).filter( + (Q(import_sources__isnull=False) & + (Q(modified_date__isnull=True) | + Q(modified_date__lte=F('imported_date')))) + & ~Q(import_sources__overlap=srcs)).values_list("import_sources", flat=True) + # get urls + urls = [url_l[0] for url_l in urls if len(url_l) > 0] + + # run tasks as a chain + tasks = chain(import_events_from_url.s(urls, force=True) if i == 0 else import_events_from_url.s(force=True) for i in range(len(urls))) + tasks.delay() + + + app.conf.beat_schedule = { + "daily_orphans_update": { + "task": "agenda_culturel.celery.update_orphan_pure_import_events", + # Daily imports at 3:14 a.m. + "schedule": crontab(hour=2, minute=22), + }, "daily_imports": { "task": "agenda_culturel.celery.daily_imports", # Daily imports at 3:14 a.m. diff --git a/src/agenda_culturel/import_tasks/extractor.py b/src/agenda_culturel/import_tasks/extractor.py index ded6b8f..dceb7ec 100644 --- a/src/agenda_culturel/import_tasks/extractor.py +++ b/src/agenda_culturel/import_tasks/extractor.py @@ -11,6 +11,7 @@ class Extractor(ABC): class Warning(IntEnum): NO_TITLE = 1 NO_START_DATE = 2 + NOT_FOUND = 3 url_referer=None @@ -204,6 +205,7 @@ class Extractor(ABC): published=False, image=None, image_alt=None, + not_found=False ): comments = '' warnings = [] @@ -217,6 +219,8 @@ class Extractor(ABC): published = False start_day = datetime.now().date().strftime("%Y-%m-%d") warnings.append(Extractor.Warning.NO_START_DATE) + if not_found: + warnings.append(Extractor.Warning.NOT_FOUND) tags_default = self.default_value_if_exists(default_values, "tags") if not tags_default: @@ -306,7 +310,7 @@ class EventNotFoundExtractor(Extractor): self.add_event(default_values, "événement sans titre depuis " + url, None, timezone.now().date(), None, "l'import a échoué, la saisie doit se faire manuellement à partir de l'url source " + url, - [], [url], published=False, url_human=url) + [], [url], published=False, url_human=url, not_found=True) return self.get_structure() diff --git a/src/agenda_culturel/import_tasks/extractor_facebook.py b/src/agenda_culturel/import_tasks/extractor_facebook.py index 9b249a3..9563c68 100644 --- a/src/agenda_culturel/import_tasks/extractor_facebook.py +++ b/src/agenda_culturel/import_tasks/extractor_facebook.py @@ -252,7 +252,7 @@ class FacebookEventExtractor(Extractor): def clean_url(url): - if FacebookEventExtractor.is_known_url(url): + if FacebookEventExtractor.is_known_url(url, False): u = urlparse(url) result = "https://www.facebook.com" + u.path @@ -269,9 +269,12 @@ class FacebookEventExtractor(Extractor): return url - def is_known_url(url): + def is_known_url(url, include_links=True): u = urlparse(url) - return u.netloc in ["facebook.com", "www.facebook.com", "m.facebook.com"] + url_list = ["facebook.com", "www.facebook.com", "m.facebook.com"] + if include_links: + url_list.append("fb.me") + return u.netloc in url_list def extract( self, content, url, url_human=None, default_values=None, published=False diff --git a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po index 2812590..ed93739 100644 --- a/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po +++ b/src/agenda_culturel/locale/fr/LC_MESSAGES/django.po @@ -2,13 +2,13 @@ # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER # This file is distributed under the same license as the PACKAGE package. # Jean-Marie Favreau , 2023. -# +# #, fuzzy msgid "" msgstr "" "Project-Id-Version: agenda_culturel\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2025-02-07 16:00+0100\n" +"POT-Creation-Date: 2025-02-09 13:22+0100\n" "PO-Revision-Date: 2023-10-29 14:16+0000\n" "Last-Translator: Jean-Marie Favreau \n" "Language-Team: Jean-Marie Favreau \n" @@ -121,11 +121,11 @@ msgid "Imported from" msgstr "Importé depuis" #: agenda_culturel/filters.py:397 agenda_culturel/models.py:648 -#: agenda_culturel/models.py:2272 +#: agenda_culturel/models.py:2284 msgid "Status" msgstr "Status" -#: agenda_culturel/filters.py:398 agenda_culturel/models.py:2058 +#: agenda_culturel/filters.py:398 agenda_culturel/models.py:2069 msgid "Closed" msgstr "Fermé" @@ -134,7 +134,7 @@ msgid "Open" msgstr "Ouvert" #: agenda_culturel/filters.py:402 agenda_culturel/filters.py:403 -#: agenda_culturel/models.py:2052 +#: agenda_culturel/models.py:2063 msgid "Spam" msgstr "Spam" @@ -142,7 +142,7 @@ msgstr "Spam" msgid "Non spam" msgstr "Non spam" -#: agenda_culturel/filters.py:408 agenda_culturel/models.py:2073 +#: agenda_culturel/filters.py:408 agenda_culturel/models.py:2084 msgid "Type" msgstr "Type" @@ -173,11 +173,11 @@ msgstr "" msgid "Your email" msgstr "Votre adresse email" -#: agenda_culturel/forms.py:138 agenda_culturel/models.py:2042 +#: agenda_culturel/forms.py:138 agenda_culturel/models.py:2053 msgid "Your email address" msgstr "Votre adresse email" -#: agenda_culturel/forms.py:144 agenda_culturel/models.py:2065 +#: agenda_culturel/forms.py:144 agenda_culturel/models.py:2076 msgid "Comments" msgstr "Commentaires" @@ -193,8 +193,8 @@ msgid "Receive notification of publication or leave a message for moderation" msgstr "Être notifié de la publication ou laisser un message à la modération" #: agenda_culturel/forms.py:174 agenda_culturel/models.py:188 -#: agenda_culturel/models.py:653 agenda_culturel/models.py:2194 -#: agenda_culturel/models.py:2304 +#: agenda_culturel/models.py:653 agenda_culturel/models.py:2206 +#: agenda_culturel/models.py:2316 msgid "Category" msgstr "Catégorie" @@ -230,7 +230,7 @@ msgid "Details" msgstr "Détails" #: agenda_culturel/forms.py:327 agenda_culturel/models.py:683 -#: agenda_culturel/models.py:2169 +#: agenda_culturel/models.py:2181 msgid "Location" msgstr "Localisation" @@ -324,7 +324,7 @@ msgid "Apply category {} to the event {}" msgstr "Appliquer la catégorie {} à l'événement {}" #: agenda_culturel/forms.py:749 agenda_culturel/models.py:495 -#: agenda_culturel/models.py:2356 +#: agenda_culturel/models.py:2368 msgid "Place" msgstr "Lieu" @@ -362,7 +362,7 @@ msgstr "Informations" msgid "Add a comment" msgstr "Ajouter un commentaire" -#: agenda_culturel/import_tasks/custom_extractors/fbevents.py:60 +#: agenda_culturel/import_tasks/custom_extractors/fbevents.py:69 msgid "" "the page was not yet populated with events, so the loading time was probably " "too short" @@ -370,18 +370,18 @@ msgstr "" "la page n'était pas encore peuplée des événements, le temps de chargement a " "sans doute été trop court" -#: agenda_culturel/import_tasks/custom_extractors/fbevents.py:101 +#: agenda_culturel/import_tasks/custom_extractors/fbevents.py:110 msgid "Cannot get Facebook event from {}" msgstr "Impossible de récupérer un événement Facebook depuis {}" -#: agenda_culturel/import_tasks/extractor.py:202 +#: agenda_culturel/import_tasks/extractor.py:215 msgid "Unknown title" msgstr "Titre inconnu" #: agenda_culturel/models.py:67 agenda_culturel/models.py:115 #: agenda_culturel/models.py:198 agenda_culturel/models.py:438 #: agenda_culturel/models.py:466 agenda_culturel/models.py:553 -#: agenda_culturel/models.py:2034 agenda_culturel/models.py:2123 +#: agenda_culturel/models.py:2045 agenda_culturel/models.py:2135 msgid "Name" msgstr "Nom" @@ -618,7 +618,7 @@ msgstr "Organisme" msgid "Organisations" msgstr "Organismes" -#: agenda_culturel/models.py:594 agenda_culturel/models.py:2164 +#: agenda_culturel/models.py:594 agenda_culturel/models.py:2176 msgid "Published" msgstr "Publié" @@ -746,7 +746,7 @@ msgstr "Ton événement a été publié" msgid "Your message has not been retained" msgstr "Ton événement n'a pas été retenu" -#: agenda_culturel/models.py:1332 agenda_culturel/models.py:1996 +#: agenda_culturel/models.py:1332 agenda_culturel/models.py:2007 msgid "Warning" msgstr "Warning" @@ -759,6 +759,7 @@ msgid "during import process" msgstr "pendant le processus d'import" #: agenda_culturel/models.py:1420 agenda_culturel/models.py:1426 +#: agenda_culturel/models.py:1431 msgid "warning" msgstr "attention" @@ -766,27 +767,31 @@ msgstr "attention" msgid "the title has not been imported correctly." msgstr "le titre n'a pas été importé correctement." -#: agenda_culturel/models.py:1742 +#: agenda_culturel/models.py:1433 +msgid "The import was unable to find an event in the page." +msgstr "L'import a été incapable de trouver un événement dans la page." + +#: agenda_culturel/models.py:1753 msgid "Updated field(s): " msgstr "Champ(s) mis à jour: " -#: agenda_culturel/models.py:1743 +#: agenda_culturel/models.py:1754 msgid "Update" msgstr "Mise à jour" -#: agenda_culturel/models.py:1744 +#: agenda_culturel/models.py:1755 msgid "update process" msgstr "processus de mise à jour" -#: agenda_culturel/models.py:1788 +#: agenda_culturel/models.py:1799 msgid "Import" msgstr "Import" -#: agenda_culturel/models.py:1789 +#: agenda_culturel/models.py:1800 msgid "import process" msgstr "processus d'import" -#: agenda_culturel/models.py:1790 +#: agenda_culturel/models.py:1801 msgid "" "The duration of the event is a little too long for direct publication. " "Moderators can choose to publish it or not." @@ -794,175 +799,179 @@ msgstr "" "La durée de l'événement est un peu trop longue pour qu'il soit publié " "directement. Les modérateurs peuvent choisir de le publier ou non." -#: agenda_culturel/models.py:1990 +#: agenda_culturel/models.py:2001 msgid "From contributor" msgstr "D'un·e contributeurice" -#: agenda_culturel/models.py:1991 +#: agenda_culturel/models.py:2002 msgid "Import process" msgstr "Processus d'import" -#: agenda_culturel/models.py:1992 +#: agenda_culturel/models.py:2003 msgid "Update process" msgstr "Processus de mise à jour" -#: agenda_culturel/models.py:1993 +#: agenda_culturel/models.py:2004 msgid "Contact form" msgstr "Formulaire de contact" -#: agenda_culturel/models.py:1994 +#: agenda_culturel/models.py:2005 msgid "Event report" msgstr "Signalemet d'événement" -#: agenda_culturel/models.py:1995 +#: agenda_culturel/models.py:2006 msgid "From contributor (without message)" msgstr "D'un·e contributeurice (sans message)" -#: agenda_culturel/models.py:1999 agenda_culturel/models.py:2047 +#: agenda_culturel/models.py:2010 agenda_culturel/models.py:2058 msgid "Message" msgstr "Message" -#: agenda_culturel/models.py:2000 +#: agenda_culturel/models.py:2011 msgid "Messages" msgstr "Messages" -#: agenda_culturel/models.py:2011 +#: agenda_culturel/models.py:2022 msgid "Subject" msgstr "Sujet" -#: agenda_culturel/models.py:2012 +#: agenda_culturel/models.py:2023 msgid "The subject of your message" msgstr "Sujet de votre message" -#: agenda_culturel/models.py:2018 +#: agenda_culturel/models.py:2029 msgid "Related event" msgstr "Événement associé" -#: agenda_culturel/models.py:2019 +#: agenda_culturel/models.py:2030 msgid "The message is associated with this event." msgstr "Le message est associé à cet événement." -#: agenda_culturel/models.py:2027 +#: agenda_culturel/models.py:2038 msgid "Author of the message" msgstr "Auteur du message" -#: agenda_culturel/models.py:2035 +#: agenda_culturel/models.py:2046 msgid "Your name" msgstr "Votre nom" -#: agenda_culturel/models.py:2041 +#: agenda_culturel/models.py:2052 msgid "Email address" msgstr "Adresse email" -#: agenda_culturel/models.py:2047 +#: agenda_culturel/models.py:2058 msgid "Your message" msgstr "Votre message" -#: agenda_culturel/models.py:2053 +#: agenda_culturel/models.py:2064 msgid "This message is a spam." msgstr "Ce message est un spam." -#: agenda_culturel/models.py:2060 +#: agenda_culturel/models.py:2071 msgid "this message has been processed and no longer needs to be handled" msgstr "Ce message a été traité et ne nécessite plus d'être pris en charge" -#: agenda_culturel/models.py:2066 +#: agenda_culturel/models.py:2077 msgid "Comments on the message from the moderation team" msgstr "Commentaires sur ce message par l'équipe de modération" -#: agenda_culturel/models.py:2088 agenda_culturel/models.py:2252 +#: agenda_culturel/models.py:2099 agenda_culturel/models.py:2264 msgid "Recurrent import" msgstr "Import récurrent" -#: agenda_culturel/models.py:2089 +#: agenda_culturel/models.py:2100 msgid "Recurrent imports" msgstr "Imports récurrents" -#: agenda_culturel/models.py:2093 +#: agenda_culturel/models.py:2104 msgid "ical" msgstr "ical" -#: agenda_culturel/models.py:2094 +#: agenda_culturel/models.py:2105 msgid "ical no busy" msgstr "ical sans busy" -#: agenda_culturel/models.py:2095 +#: agenda_culturel/models.py:2106 msgid "ical no VC" msgstr "ical sans VC" -#: agenda_culturel/models.py:2096 +#: agenda_culturel/models.py:2107 msgid "lacoope.org" msgstr "lacoope.org" -#: agenda_culturel/models.py:2097 +#: agenda_culturel/models.py:2108 msgid "la comédie" msgstr "la comédie" -#: agenda_culturel/models.py:2098 +#: agenda_culturel/models.py:2109 msgid "le fotomat" msgstr "le fotomat" -#: agenda_culturel/models.py:2099 +#: agenda_culturel/models.py:2110 msgid "la puce à l'oreille" msgstr "la puce à loreille" -#: agenda_culturel/models.py:2100 +#: agenda_culturel/models.py:2111 msgid "Plugin wordpress MEC" msgstr "Plugin wordpress MEC" -#: agenda_culturel/models.py:2101 +#: agenda_culturel/models.py:2112 msgid "Événements d'une page FB" msgstr "Événements d'une page FB" -#: agenda_culturel/models.py:2102 +#: agenda_culturel/models.py:2113 msgid "Billetterie Clermont-Ferrand" msgstr "" -#: agenda_culturel/models.py:2103 +#: agenda_culturel/models.py:2114 msgid "Arachnée concert" msgstr "Arachnée concert" -#: agenda_culturel/models.py:2104 +#: agenda_culturel/models.py:2115 msgid "Le Rio" msgstr "Le Rio" -#: agenda_culturel/models.py:2105 +#: agenda_culturel/models.py:2116 msgid "La Raymonde" msgstr "La Raymone" -#: agenda_culturel/models.py:2106 +#: agenda_culturel/models.py:2117 msgid "Agenda apidae tourisme" msgstr "Agenda apidae tourisme" -#: agenda_culturel/models.py:2107 +#: agenda_culturel/models.py:2118 msgid "Agenda iguana (médiathèques)" msgstr "Agenda iguana (médiathèques)" -#: agenda_culturel/models.py:2108 +#: agenda_culturel/models.py:2119 msgid "Mille formes" -msgstr "" +msgstr "Mille Formes" -#: agenda_culturel/models.py:2111 +#: agenda_culturel/models.py:2120 +msgid "Les Amis du Temps des Cerises" +msgstr "Les Amis du Temps des Cerises" + +#: agenda_culturel/models.py:2123 msgid "simple" msgstr "simple" -#: agenda_culturel/models.py:2112 +#: agenda_culturel/models.py:2124 msgid "Headless Chromium" msgstr "chromium sans interface" -#: agenda_culturel/models.py:2113 +#: agenda_culturel/models.py:2125 msgid "Headless Chromium (pause)" msgstr "chromium sans interface (pause)" -#: agenda_culturel/models.py:2118 +#: agenda_culturel/models.py:2130 msgid "daily" msgstr "chaque jour" -#: agenda_culturel/models.py:2120 +#: agenda_culturel/models.py:2132 msgid "weekly" msgstr "chaque semaine" -#: agenda_culturel/models.py:2125 +#: agenda_culturel/models.py:2137 msgid "" "Recurrent import name. Be careful to choose a name that is easy to " "understand, as it will be public and displayed on the sites About page." @@ -970,151 +979,151 @@ msgstr "" "Nom de l'import récurrent. Attention à choisir un nom compréhensible, car il " "sera public, et affiché sur la page à propos du site." -#: agenda_culturel/models.py:2132 +#: agenda_culturel/models.py:2144 msgid "Processor" msgstr "Processeur" -#: agenda_culturel/models.py:2135 +#: agenda_culturel/models.py:2147 msgid "Downloader" msgstr "Téléchargeur" -#: agenda_culturel/models.py:2142 +#: agenda_culturel/models.py:2154 msgid "Import recurrence" msgstr "Récurrence d'import" -#: agenda_culturel/models.py:2149 +#: agenda_culturel/models.py:2161 msgid "Source" msgstr "Source" -#: agenda_culturel/models.py:2150 +#: agenda_culturel/models.py:2162 msgid "URL of the source document" msgstr "URL du document source" -#: agenda_culturel/models.py:2154 +#: agenda_culturel/models.py:2166 msgid "Browsable url" msgstr "URL navigable" -#: agenda_culturel/models.py:2156 +#: agenda_culturel/models.py:2168 msgid "URL of the corresponding document that will be shown to visitors." msgstr "URL correspondant au document et qui sera montrée aux visiteurs" -#: agenda_culturel/models.py:2165 +#: agenda_culturel/models.py:2177 msgid "Status of each imported event (published or draft)" msgstr "Status de chaque événement importé (publié ou brouillon)" -#: agenda_culturel/models.py:2170 +#: agenda_culturel/models.py:2182 msgid "Address for each imported event" msgstr "Adresse de chaque événement importé" -#: agenda_culturel/models.py:2177 +#: agenda_culturel/models.py:2189 msgid "Force location" msgstr "Focer la localisation" -#: agenda_culturel/models.py:2178 +#: agenda_culturel/models.py:2190 msgid "force location even if another is detected." msgstr "Forcer la localisation même si une autre a été détectée." -#: agenda_culturel/models.py:2184 +#: agenda_culturel/models.py:2196 msgid "Organiser" msgstr "Organisateur" -#: agenda_culturel/models.py:2185 +#: agenda_culturel/models.py:2197 msgid "Organiser of each imported event" msgstr "Organisateur de chaque événement importé" -#: agenda_culturel/models.py:2195 +#: agenda_culturel/models.py:2207 msgid "Category of each imported event" msgstr "Catégorie de chaque événement importé" -#: agenda_culturel/models.py:2203 +#: agenda_culturel/models.py:2215 msgid "Tags for each imported event" msgstr "Étiquettes de chaque événement importé" -#: agenda_culturel/models.py:2204 +#: agenda_culturel/models.py:2216 msgid "A list of tags that describe each imported event." msgstr "Une liste d'étiquettes décrivant chaque événement importé" -#: agenda_culturel/models.py:2233 +#: agenda_culturel/models.py:2245 msgid "Running" msgstr "En cours" -#: agenda_culturel/models.py:2234 +#: agenda_culturel/models.py:2246 msgid "Canceled" msgstr "Annulé" -#: agenda_culturel/models.py:2235 +#: agenda_culturel/models.py:2247 msgid "Success" msgstr "Succès" -#: agenda_culturel/models.py:2236 +#: agenda_culturel/models.py:2248 msgid "Failed" msgstr "Erreur" -#: agenda_culturel/models.py:2239 +#: agenda_culturel/models.py:2251 msgid "Batch importation" msgstr "Importation par lot" -#: agenda_culturel/models.py:2240 +#: agenda_culturel/models.py:2252 msgid "Batch importations" msgstr "Importations par lot" -#: agenda_culturel/models.py:2253 +#: agenda_culturel/models.py:2265 msgid "Reference to the recurrent import processing" msgstr "Référence du processus d'import récurrent" -#: agenda_culturel/models.py:2261 +#: agenda_culturel/models.py:2273 msgid "URL (if not recurrent import)" msgstr "URL (si pas d'import récurrent)" -#: agenda_culturel/models.py:2263 +#: agenda_culturel/models.py:2275 msgid "Source URL if no RecurrentImport is associated." msgstr "URL source si aucun import récurrent n'est associé" -#: agenda_culturel/models.py:2276 +#: agenda_culturel/models.py:2288 msgid "Error message" msgstr "Votre message" -#: agenda_culturel/models.py:2280 +#: agenda_culturel/models.py:2292 msgid "Number of collected events" msgstr "Nombre d'événements collectés" -#: agenda_culturel/models.py:2283 +#: agenda_culturel/models.py:2295 msgid "Number of imported events" msgstr "Nombre d'événements importés" -#: agenda_culturel/models.py:2286 +#: agenda_culturel/models.py:2298 msgid "Number of updated events" msgstr "Nombre d'événements mis à jour" -#: agenda_culturel/models.py:2289 +#: agenda_culturel/models.py:2301 msgid "Number of removed events" msgstr "Nombre d'événements supprimés" -#: agenda_culturel/models.py:2297 +#: agenda_culturel/models.py:2309 msgid "Weight" msgstr "Poids" -#: agenda_culturel/models.py:2298 +#: agenda_culturel/models.py:2310 msgid "The lower is the weight, the earlier the filter is applied" msgstr "Plus le poids est léger, plus le filtre sera appliqué tôt" -#: agenda_culturel/models.py:2305 +#: agenda_culturel/models.py:2317 msgid "Category applied to the event" msgstr "Catégorie appliquée à l'événement" -#: agenda_culturel/models.py:2310 +#: agenda_culturel/models.py:2322 msgid "Contained in the title" msgstr "Contenu dans le titre" -#: agenda_culturel/models.py:2311 +#: agenda_culturel/models.py:2323 msgid "Text contained in the event title" msgstr "Texte contenu dans le titre de l'événement" -#: agenda_culturel/models.py:2317 +#: agenda_culturel/models.py:2329 msgid "Exact title extract" msgstr "Extrait exact du titre" -#: agenda_culturel/models.py:2319 +#: agenda_culturel/models.py:2331 msgid "" "If checked, the extract will be searched for in the title using the exact " "form (capitals, accents)." @@ -1122,19 +1131,19 @@ msgstr "" "Si coché, l'extrait sera recherché dans le titre en utilisant la forme " "exacte (majuscules, accents)" -#: agenda_culturel/models.py:2325 +#: agenda_culturel/models.py:2337 msgid "Contained in the description" msgstr "Contenu dans la description" -#: agenda_culturel/models.py:2326 +#: agenda_culturel/models.py:2338 msgid "Text contained in the description" msgstr "Texte contenu dans la description" -#: agenda_culturel/models.py:2332 +#: agenda_culturel/models.py:2344 msgid "Exact description extract" msgstr "Extrait exact de description" -#: agenda_culturel/models.py:2334 +#: agenda_culturel/models.py:2346 msgid "" "If checked, the extract will be searched for in the description using the " "exact form (capitals, accents)." @@ -1142,19 +1151,19 @@ msgstr "" "Si coché, l'extrait sera recherché dans la description en utilisant la forme " "exacte (majuscules, accents)" -#: agenda_culturel/models.py:2340 +#: agenda_culturel/models.py:2352 msgid "Contained in the location" msgstr "Contenu dans la localisation" -#: agenda_culturel/models.py:2341 +#: agenda_culturel/models.py:2353 msgid "Text contained in the event location" msgstr "Texte contenu dans la localisation de l'événement" -#: agenda_culturel/models.py:2347 +#: agenda_culturel/models.py:2359 msgid "Exact location extract" msgstr "Extrait exact de localisation" -#: agenda_culturel/models.py:2349 +#: agenda_culturel/models.py:2361 msgid "" "If checked, the extract will be searched for in the location using the exact " "form (capitals, accents)." @@ -1162,15 +1171,15 @@ msgstr "" "Si coché, l'extrait sera recherché dans la localisation en utilisant la " "forme exacte (majuscules, accents)" -#: agenda_culturel/models.py:2357 +#: agenda_culturel/models.py:2369 msgid "Location from place" msgstr "Localisation depuis le lieu" -#: agenda_culturel/models.py:2366 +#: agenda_culturel/models.py:2378 msgid "Categorisation rule" msgstr "Règle de catégorisation" -#: agenda_culturel/models.py:2367 +#: agenda_culturel/models.py:2379 msgid "Categorisation rules" msgstr "Règles de catégorisation" @@ -1178,35 +1187,35 @@ msgstr "Règles de catégorisation" msgid "French" msgstr "français" -#: agenda_culturel/views.py:156 +#: agenda_culturel/views.py:157 msgid "Recurrent import name" msgstr "Nome de l'import récurrent" -#: agenda_culturel/views.py:157 +#: agenda_culturel/views.py:158 msgid "Add another" msgstr "Ajouter un autre" -#: agenda_culturel/views.py:158 +#: agenda_culturel/views.py:159 msgid "Browse..." msgstr "Naviguer..." -#: agenda_culturel/views.py:159 +#: agenda_culturel/views.py:160 msgid "No file selected." msgstr "Pas de fichier sélectionné." -#: agenda_culturel/views.py:175 +#: agenda_culturel/views.py:176 msgid ": error 500" msgstr ": erreur 500" -#: agenda_culturel/views.py:176 +#: agenda_culturel/views.py:177 msgid "An internal error has occurred on site {} at address {}." msgstr "Une erreur interne s'est produite sur le site {} à l'adresse {}." -#: agenda_culturel/views.py:330 +#: agenda_culturel/views.py:331 msgid "The static content has been successfully updated." msgstr "Le contenu statique a été modifié avec succès." -#: agenda_culturel/views.py:338 +#: agenda_culturel/views.py:339 msgid "" "The event cannot be updated because the import process is not available for " "the referenced sources." @@ -1214,21 +1223,21 @@ msgstr "" "La mise à jour de l'événement n'est pas possible car le processus d'import " "n'est pas disponible pour les sources référencées." -#: agenda_culturel/views.py:341 +#: agenda_culturel/views.py:342 msgid "The event update has been queued and will be completed shortly." msgstr "" "La mise à jour de l'événement a été mise en attente et sera effectuée sous " "peu." -#: agenda_culturel/views.py:360 agenda_culturel/views.py:415 +#: agenda_culturel/views.py:361 agenda_culturel/views.py:416 msgid " A message has been sent to the person who proposed the event." msgstr " Un message a été envoyé à la personne qui a proposé l'événement." -#: agenda_culturel/views.py:361 +#: agenda_culturel/views.py:362 msgid "The event has been successfully modified." msgstr "L'événement a été modifié avec succès." -#: agenda_culturel/views.py:377 +#: agenda_culturel/views.py:378 msgid "" "Changes will be visible on a local copy of the event. The version identical " "to the imported source will be hidden." @@ -1236,19 +1245,19 @@ msgstr "" "Les modifications seront visibles sur une copie locale de l'événement. La " "version fidèle à la source importée sera masquée." -#: agenda_culturel/views.py:416 +#: agenda_culturel/views.py:417 msgid "The event {} has been moderated with success." msgstr "L'événement {} a été modéré avec succès." -#: agenda_culturel/views.py:535 +#: agenda_culturel/views.py:536 msgid "The event has been successfully deleted." msgstr "L'événement a été supprimé avec succès." -#: agenda_culturel/views.py:577 +#: agenda_culturel/views.py:581 msgid "Comment" msgstr "Commentaire" -#: agenda_culturel/views.py:602 +#: agenda_culturel/views.py:606 msgid "" "The status has been successfully modified and a message has been sent to the " "person who proposed the event." @@ -1256,15 +1265,15 @@ msgstr "" "Le status a été modifié avec succès et un message a été envoyé à la personne " "qui a proposé l'événement." -#: agenda_culturel/views.py:604 +#: agenda_culturel/views.py:608 msgid "The status has been successfully modified." msgstr "Le status a été modifié avec succès." -#: agenda_culturel/views.py:638 +#: agenda_culturel/views.py:642 msgid "The event was created: {}." msgstr "L'événement a été créé: {}." -#: agenda_culturel/views.py:640 +#: agenda_culturel/views.py:644 msgid "" "The event has been submitted and will be published as soon as it has been " "validated by the moderation team." @@ -1272,91 +1281,95 @@ msgstr "" "L'événement a été soumis et sera publié dès qu'il aura été validé par " "l'équipe de modération." -#: agenda_culturel/views.py:654 +#: agenda_culturel/views.py:658 msgid "during the creation process" msgstr "pendant le processus d'import" -#: agenda_culturel/views.py:668 +#: agenda_culturel/views.py:672 msgid "A message has been sent to the person who proposed the initial event." msgstr "" "Un message a été envoyé à la personne qui a proposé l'événement initial." -#: agenda_culturel/views.py:756 agenda_culturel/views.py:820 +#: agenda_culturel/views.py:760 agenda_culturel/views.py:824 msgid "{} has not been submitted since its already known: {}." msgstr "{} n'a pas été soumis car il est déjà connu: {}." -#: agenda_culturel/views.py:761 agenda_culturel/views.py:826 +#: agenda_culturel/views.py:765 agenda_culturel/views.py:830 msgid "" "{} has not been submitted since its already known and currently into " "moderation process." msgstr "{} n'a pas été soumis car il est déjà connu et en cours de modération" -#: agenda_culturel/views.py:771 +#: agenda_culturel/views.py:775 msgid "Integrating {} url(s) into our import process." msgstr "Intégration de {} url(s) dans notre processus d'import." -#: agenda_culturel/views.py:833 +#: agenda_culturel/views.py:837 msgid "Integrating {} into our import process." msgstr "Intégration de {} dans notre processus d'import." -#: agenda_culturel/views.py:910 +#: agenda_culturel/views.py:914 msgid "Your message has been sent successfully." msgstr "Votre message a été envoyé avec succès." -#: agenda_culturel/views.py:940 +#: agenda_culturel/views.py:944 msgid "Reporting the event {} on {}" msgstr "Signaler l'événement {} du {}" -#: agenda_culturel/views.py:950 +#: agenda_culturel/views.py:954 msgid "The contact message has been successfully deleted." msgstr "Le message de contact a été supprimé avec succès." -#: agenda_culturel/views.py:964 +#: agenda_culturel/views.py:968 msgid "The contact message properties has been successfully modified." msgstr "Les propriétés du message de contact ont été modifié avec succès." -#: agenda_culturel/views.py:1110 +#: agenda_culturel/views.py:1125 msgid "Spam has been successfully deleted." msgstr "Le spam a été supprimé avec succès" -#: agenda_culturel/views.py:1233 +#: agenda_culturel/views.py:1260 msgid "The import has been run successfully." msgstr "L'import a été lancé avec succès" -#: agenda_culturel/views.py:1252 +#: agenda_culturel/views.py:1279 msgid "The import has been canceled." msgstr "L'import a été annulé" -#: agenda_culturel/views.py:1330 +#: agenda_culturel/views.py:1298 +msgid "The orphan event update has been launched." +msgstr "La mise à jour de l'événement orphelin a été lancée." + +#: agenda_culturel/views.py:1382 msgid "The recurrent import has been successfully modified." msgstr "L'import récurrent a été modifié avec succès." -#: agenda_culturel/views.py:1339 +#: agenda_culturel/views.py:1391 msgid "The recurrent import has been successfully deleted." msgstr "L'import récurrent a été supprimé avec succès" -#: agenda_culturel/views.py:1379 +#: agenda_culturel/views.py:1431 msgid "The import has been launched." msgstr "L'import a été lancé" -#: agenda_culturel/views.py:1401 +#: agenda_culturel/views.py:1453 msgid "Imports has been launched." msgstr "Les imports ont été lancés" -#: agenda_culturel/views.py:1463 +#: agenda_culturel/views.py:1515 msgid "Update successfully completed." msgstr "Mise à jour réalisée avec succès." -#: agenda_culturel/views.py:1524 +#: agenda_culturel/views.py:1576 msgid "Creation of a merged event has been successfully completed." msgstr "Création d'un événement fusionné réalisée avec succès." -#: agenda_culturel/views.py:1560 +#: agenda_culturel/views.py:1612 msgid "Events have been marked as unduplicated." msgstr "Les événements ont été marqués comme non dupliqués." -#: agenda_culturel/views.py:1574 agenda_culturel/views.py:1583 -#: agenda_culturel/views.py:1601 +#: agenda_culturel/views.py:1626 agenda_culturel/views.py:1635 +#: agenda_culturel/views.py:1653 msgid "" "The selected item is no longer included in the list of duplicates. Someone " "else has probably modified the list in the meantime." @@ -1364,23 +1377,23 @@ msgstr "" "L'élément sélectionné ne fait plus partie de la liste des dupliqués. Une " "autre personne a probablement modifié la liste entre temps." -#: agenda_culturel/views.py:1577 +#: agenda_culturel/views.py:1629 msgid "The selected event has been set as representative" msgstr "L'événement sélectionné a été défini comme representatif." -#: agenda_culturel/views.py:1592 +#: agenda_culturel/views.py:1644 msgid "The event has been withdrawn from the group and made independent." msgstr "L'événement a été retiré du groupe et rendu indépendant." -#: agenda_culturel/views.py:1636 +#: agenda_culturel/views.py:1688 msgid "Cleaning up duplicates: {} item(s) fixed." msgstr "Nettoyage des dupliqués: {} élément(s) corrigé(s)." -#: agenda_culturel/views.py:1686 +#: agenda_culturel/views.py:1738 msgid "The event was successfully duplicated." msgstr "L'événement a été marqué dupliqué avec succès." -#: agenda_culturel/views.py:1694 +#: agenda_culturel/views.py:1746 msgid "" "The event has been successfully flagged as a duplicate. The moderation team " "will deal with your suggestion shortly." @@ -1388,32 +1401,32 @@ msgstr "" "L'événement a été signalé comme dupliqué avec succès. Votre suggestion sera " "prochainement prise en charge par l'équipe de modération." -#: agenda_culturel/views.py:1747 +#: agenda_culturel/views.py:1799 msgid "The categorisation rule has been successfully modified." msgstr "La règle de catégorisation a été modifiée avec succès." -#: agenda_culturel/views.py:1756 +#: agenda_culturel/views.py:1808 msgid "The categorisation rule has been successfully deleted." msgstr "La règle de catégorisation a été supprimée avec succès" -#: agenda_culturel/views.py:1778 +#: agenda_culturel/views.py:1830 msgid "The rules were successfully applied and 1 event was categorised." msgstr "" "Les règles ont été appliquées avec succès et 1 événement a été catégorisé" -#: agenda_culturel/views.py:1785 +#: agenda_culturel/views.py:1837 msgid "The rules were successfully applied and {} events were categorised." msgstr "" "Les règles ont été appliquées avec succès et {} événements ont été " "catégorisés" -#: agenda_culturel/views.py:1792 agenda_culturel/views.py:1845 +#: agenda_culturel/views.py:1844 agenda_culturel/views.py:1897 msgid "The rules were successfully applied and no events were categorised." msgstr "" "Les règles ont été appliquées avec succès et aucun événement n'a été " "catégorisé" -#: agenda_culturel/views.py:1831 +#: agenda_culturel/views.py:1883 msgid "" "The rules were successfully applied and 1 event with default category was " "categorised." @@ -1421,7 +1434,7 @@ msgstr "" "Les règles ont été appliquées avec succès et 1 événement avec catégorie par " "défaut a été catégorisé" -#: agenda_culturel/views.py:1838 +#: agenda_culturel/views.py:1890 msgid "" "The rules were successfully applied and {} events with default category were " "categorised." @@ -1429,58 +1442,58 @@ msgstr "" "Les règles ont été appliquées avec succès et {} événements avec catégorie " "par défaut ont été catégorisés" -#: agenda_culturel/views.py:1930 agenda_culturel/views.py:1992 -#: agenda_culturel/views.py:2030 +#: agenda_culturel/views.py:1982 agenda_culturel/views.py:2044 +#: agenda_culturel/views.py:2082 msgid "{} events have been updated." msgstr "{} événements ont été mis à jour." -#: agenda_culturel/views.py:1933 agenda_culturel/views.py:1994 -#: agenda_culturel/views.py:2033 +#: agenda_culturel/views.py:1985 agenda_culturel/views.py:2046 +#: agenda_culturel/views.py:2085 msgid "1 event has been updated." msgstr "1 événement a été mis à jour" -#: agenda_culturel/views.py:1935 agenda_culturel/views.py:1996 -#: agenda_culturel/views.py:2035 +#: agenda_culturel/views.py:1987 agenda_culturel/views.py:2048 +#: agenda_culturel/views.py:2087 msgid "No events have been modified." msgstr "Aucun événement n'a été modifié." -#: agenda_culturel/views.py:1944 +#: agenda_culturel/views.py:1996 msgid "The place has been successfully updated." msgstr "Le lieu a été modifié avec succès." -#: agenda_culturel/views.py:1953 +#: agenda_culturel/views.py:2005 msgid "The place has been successfully created." msgstr "Le lieu a été créé avec succès." -#: agenda_culturel/views.py:2018 +#: agenda_culturel/views.py:2070 msgid "The selected place has been assigned to the event." msgstr "Le lieu sélectionné a été assigné à l'événement." -#: agenda_culturel/views.py:2022 +#: agenda_culturel/views.py:2074 msgid "A new alias has been added to the selected place." msgstr "Un nouvel alias a été créé pour le lieu sélectionné." -#: agenda_culturel/views.py:2124 +#: agenda_culturel/views.py:2176 msgid "The organisation has been successfully updated." msgstr "L'organisme a été modifié avec succès." -#: agenda_culturel/views.py:2133 +#: agenda_culturel/views.py:2185 msgid "The organisation has been successfully created." msgstr "L'organisme a été créé avec succès." -#: agenda_culturel/views.py:2150 +#: agenda_culturel/views.py:2202 msgid "The tag has been successfully updated." msgstr "L'étiquette a été modifiée avec succès." -#: agenda_culturel/views.py:2157 +#: agenda_culturel/views.py:2209 msgid "The tag has been successfully created." msgstr "L'étiquette a été créée avec succès." -#: agenda_culturel/views.py:2233 +#: agenda_culturel/views.py:2285 msgid "You have not modified the tag name." msgstr "Vous n'avez pas modifié le nom de l'étiquette." -#: agenda_culturel/views.py:2243 +#: agenda_culturel/views.py:2295 msgid "" "This tag {} is already in use, and is described by different information " "from the current tag. You can force renaming by checking the corresponding " @@ -1493,7 +1506,7 @@ msgstr "" "sera supprimée, et tous les événements associés à l'étiquette {} seront " "associés à l'étiquette {}." -#: agenda_culturel/views.py:2250 +#: agenda_culturel/views.py:2302 msgid "" "This tag {} is already in use. You can force renaming by checking the " "corresponding option." @@ -1501,15 +1514,15 @@ msgstr "" "Cette étiquette {} est déjà utilisée. Vous pouvez forcer le renommage en " "cochant l'option correspondante." -#: agenda_culturel/views.py:2284 +#: agenda_culturel/views.py:2336 msgid "The tag {} has been successfully renamed to {}." msgstr "L'étiquette {} a été renommée avec succès en {}." -#: agenda_culturel/views.py:2322 +#: agenda_culturel/views.py:2374 msgid "The tag {} has been successfully deleted." msgstr "L'événement {} a été supprimé avec succès." -#: agenda_culturel/views.py:2343 +#: agenda_culturel/views.py:2395 msgid "Cache successfully cleared." msgstr "Le cache a été vidé avec succès." diff --git a/src/agenda_culturel/models.py b/src/agenda_culturel/models.py index 33a9503..3e2dbee 100644 --- a/src/agenda_culturel/models.py +++ b/src/agenda_culturel/models.py @@ -1032,6 +1032,12 @@ class Event(models.Model): def has_pending_organisers(self): return hasattr(self, "pending_organisers") + def set_is_not_found_import(self): + self.not_found_import = True + + def is_not_found_import(self): + return hasattr(self, "not_found_import") + def set_skip_duplicate_check(self): self.skip_duplicate_check = True @@ -1235,8 +1241,6 @@ class Event(models.Model): notif = False if self.status != Event.STATUS.DRAFT: messages = self.get_contributor_message() - logger.warning("messages: ") - logger.warning(messages) if messages: for message in messages: if message and not message.closed and message.email and message.email != "": @@ -1427,6 +1431,13 @@ class Event(models.Model): closed=False, message=_('the title has not been imported correctly.'), message_type=Message.TYPE.WARNING)) + if w == Extractor.Warning.NOT_FOUND: + result.status = Event.STATUS.DRAFT + result.set_is_not_found_import() + result.add_message(Message(subject=_('warning'), + closed=False, + message=_('The import was unable to find an event in the page.'), + message_type=Message.TYPE.WARNING)) return result @@ -1529,10 +1540,16 @@ class Event(models.Model): def get_organisers(self): if self.pk: - return self.organisers.all() + if self.organisers is None: + return [] + else: + return self.organisers.all() else: if self.has_pending_organisers(): - return self.pending_organisers + if self.pending_organisers is None: + return [] + else: + return self.pending_organisers else: return [] @@ -1647,7 +1664,7 @@ class Event(models.Model): def update(self, other, all): # integrate pending organisers - if other.has_pending_organisers(): + if other.has_pending_organisers() and not other.pending_organisers is None: self.organisers.set(other.pending_organisers) logger.warning("process update " + other.title + ' ' + str(other.has_invalid_start_date())) @@ -1737,20 +1754,31 @@ class Event(models.Model): same_imported.other_versions.representative = None same_imported.other_versions.save() # add a message to explain the update - res = [r for r in Event.get_comparison([event, same_imported], all) if not r["similar"]] - if len(res) > 0: - txt = _("Updated field(s): ") + ", ".join([r["key"] for r in res]) - msg = Message(subject=_('Update'), - name=_('update process'), - related_event=same_imported, - message=txt, - message_type=Message.TYPE.UPDATE_PROCESS) - msg.save() + if not event.is_not_found_import(): + res = [r for r in Event.get_comparison([event, same_imported], all) if not r["similar"]] + if len(res) > 0: + txt = _("Updated field(s): ") + ", ".join([r["key"] for r in res]) + msg = Message(subject=_('Update'), + name=_('update process'), + related_event=same_imported, + message=txt, + message_type=Message.TYPE.UPDATE_PROCESS) + msg.save() - - # we only update local information if it's a pure import and has no moderated_date new_image = same_imported.image != event.image - same_imported.update(event, pure and same_imported.moderated_date is None) + + # if the import process was not able to found any content, change the status as draft + if event.is_not_found_import(): + if same_imported.status == Event.STATUS.PUBLISHED: + same_imported.status = Event.STATUS.TRASH + else: + # we only update local information if it's a pure import and has no moderated_date + same_imported.update(event, pure and same_imported.moderated_date is None) + # save messages + if event.has_message(): + for msg in event.get_messages(): + msg.related_event = same_imported + msg.save() same_imported.set_in_importation_process() same_imported.prepare_save() # fix missing or updated files diff --git a/src/agenda_culturel/templates/agenda_culturel/batch-imports-inc.html b/src/agenda_culturel/templates/agenda_culturel/batch-imports-inc.html index ddbd57a..1d49587 100644 --- a/src/agenda_culturel/templates/agenda_culturel/batch-imports-inc.html +++ b/src/agenda_culturel/templates/agenda_culturel/batch-imports-inc.html @@ -30,6 +30,9 @@ url {% endif %} + {% if obj.event_id %} + (événement) + {% endif %} {% endif %} {% endif %} {{ obj.status }} diff --git a/src/agenda_culturel/templates/agenda_culturel/imports.html b/src/agenda_culturel/templates/agenda_culturel/imports.html index 0484c2b..c8d0730 100644 --- a/src/agenda_culturel/templates/agenda_culturel/imports.html +++ b/src/agenda_culturel/templates/agenda_culturel/imports.html @@ -8,8 +8,6 @@ {% css_categories %} {% endblock %} -{% block ajouter-bouton %}{% block ajouter-menu %}{% endblock %}{% endblock %} - {% block sidemenu-bouton %}
  • {% picto_from_name "chevron-up" %}
  • {% picto_from_name "chevron-down" %}
    • @@ -19,10 +17,14 @@
    - Import manuel +
    + Import manuel + Mettre à jour les singletons {% picto_from_name "play-circle" %} +

    Importations par lot

    +

    Il y a actuellement {{ nb_in_orphan_import }} événements singletons, c'est-à-dire importés depuis une source mais non inclus dans un import récurrent.

    {% include "agenda_culturel/batch-imports-inc.html" with objects=paginator_filter %}