diff --git a/src/agenda_culturel/admin.py b/src/agenda_culturel/admin.py
index c1aa90d..8727544 100644
--- a/src/agenda_culturel/admin.py
+++ b/src/agenda_culturel/admin.py
@@ -1,6 +1,14 @@
from django.contrib import admin
from django import forms
-from .models import Event, Category, StaticContent, DuplicatedEvents, BatchImportation, RecurrentImport, Place
+from .models import (
+ Event,
+ Category,
+ StaticContent,
+ DuplicatedEvents,
+ BatchImportation,
+ RecurrentImport,
+ Place,
+)
from django_better_admin_arrayfield.admin.mixins import DynamicArrayMixin
from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget
from django_better_admin_arrayfield.models.fields import DynamicArrayField
@@ -16,12 +24,12 @@ admin.site.register(Place)
class URLWidget(DynamicArrayWidget):
def __init__(self, *args, **kwargs):
- kwargs['subwidget_form'] = forms.URLField()
+ kwargs["subwidget_form"] = forms.URLField()
super().__init__(*args, **kwargs)
+
@admin.register(Event)
class Eventdmin(admin.ModelAdmin, DynamicArrayMixin):
-
formfield_overrides = {
- DynamicArrayField: {'urls': URLWidget},
- }
\ No newline at end of file
+ DynamicArrayField: {"urls": URLWidget},
+ }
diff --git a/src/agenda_culturel/calendar.py b/src/agenda_culturel/calendar.py
index 7e064fc..dff1816 100644
--- a/src/agenda_culturel/calendar.py
+++ b/src/agenda_culturel/calendar.py
@@ -5,6 +5,7 @@ from django.utils import timezone
import logging
+
logger = logging.getLogger(__name__)
@@ -21,7 +22,7 @@ def daterange(start, end, step=timedelta(1)):
class DayInCalendar:
midnight = time(23, 59, 59)
- def __init__(self, d, on_requested_interval = True):
+ def __init__(self, d, on_requested_interval=True):
self.date = d
now = date.today()
self.week = d.isocalendar()[1]
@@ -35,6 +36,7 @@ class DayInCalendar:
def is_in_past(self):
return self.in_past
+
def is_today(self):
return self.today
@@ -58,8 +60,9 @@ class DayInCalendar:
if removed:
# remove empty events_by_category
- self.events_by_category = dict([(k, v) for k, v in self.events_by_category.items() if len(v) > 0])
-
+ self.events_by_category = dict(
+ [(k, v) for k, v in self.events_by_category.items() if len(v) > 0]
+ )
def add_event(self, event):
if event.contains_date(self.date):
@@ -70,24 +73,26 @@ class DayInCalendar:
self.remove_event_with_ancestor_uuid_if_exists(event)
self._add_event_internal(event)
-
def _add_event_internal(self, event):
- self.events.append(event)
- if event.category is None:
- if not "" in self.events_by_category:
- self.events_by_category[""] = []
- self.events_by_category[""].append(event)
- else:
- if not event.category.name in self.events_by_category:
- self.events_by_category[event.category.name] = []
- self.events_by_category[event.category.name].append(event)
+ self.events.append(event)
+ if event.category is None:
+ if not "" in self.events_by_category:
+ self.events_by_category[""] = []
+ self.events_by_category[""].append(event)
+ else:
+ if not event.category.name in self.events_by_category:
+ self.events_by_category[event.category.name] = []
+ self.events_by_category[event.category.name].append(event)
def filter_events(self):
- self.events.sort(key=lambda e: DayInCalendar.midnight if e.start_time is None else e.start_time)
+ self.events.sort(
+ key=lambda e: DayInCalendar.midnight
+ if e.start_time is None
+ else e.start_time
+ )
class CalendarList:
-
def __init__(self, firstdate, lastdate, filter=None, exact=False):
self.firstdate = firstdate
self.lastdate = lastdate
@@ -101,8 +106,7 @@ class CalendarList:
# start the first day of the first week
self.c_firstdate = firstdate + timedelta(days=-firstdate.weekday())
# end the last day of the last week
- self.c_lastdate = lastdate + timedelta(days=6-lastdate.weekday())
-
+ self.c_lastdate = lastdate + timedelta(days=6 - lastdate.weekday())
# create a list of DayInCalendars
self.create_calendar_days()
@@ -114,7 +118,6 @@ class CalendarList:
for i, c in self.calendar_days.items():
c.filter_events()
-
def today_in_calendar(self):
return self.firstdate <= self.now and self.lastdate >= self.now
@@ -124,53 +127,56 @@ class CalendarList:
def fill_calendar_days(self):
if self.filter is None:
from .models import Event
+
qs = Event.objects.all()
else:
qs = self.filter.qs
startdatetime = datetime.combine(self.c_firstdate, time.min)
lastdatetime = datetime.combine(self.c_lastdate, time.max)
self.events = qs.filter(
- (Q(recurrence_dtend__isnull=True) & Q(recurrence_dtstart__lte=lastdatetime)) |
- (Q(recurrence_dtend__isnull=False) & ~(Q(recurrence_dtstart__gt=lastdatetime) | Q(recurrence_dtend__lt=startdatetime)))
+ (Q(recurrence_dtend__isnull=True) & Q(recurrence_dtstart__lte=lastdatetime))
+ | (
+ Q(recurrence_dtend__isnull=False)
+ & ~(
+ Q(recurrence_dtstart__gt=lastdatetime)
+ | Q(recurrence_dtend__lt=startdatetime)
+ )
+ )
).order_by("start_time")
firstdate = datetime.fromordinal(self.c_firstdate.toordinal())
if firstdate.tzinfo is None or firstdate.tzinfo.utcoffset(firstdate) is None:
firstdate = timezone.make_aware(firstdate, timezone.get_default_timezone())
-
lastdate = datetime.fromordinal(self.c_lastdate.toordinal())
if lastdate.tzinfo is None or lastdate.tzinfo.utcoffset(lastdate) is None:
lastdate = timezone.make_aware(lastdate, timezone.get_default_timezone())
-
for e in self.events:
for e_rec in e.get_recurrences_between(firstdate, lastdate):
for d in daterange(e_rec.start_day, e_rec.end_day):
if d.__str__() in self.calendar_days:
self.calendar_days[d.__str__()].add_event(e_rec)
-
def create_calendar_days(self):
# create daylist
self.calendar_days = {}
for d in daterange(self.c_firstdate, self.c_lastdate):
- self.calendar_days[d.strftime("%Y-%m-%d")] = DayInCalendar(d, d >= self.firstdate and d <= self.lastdate)
-
+ self.calendar_days[d.strftime("%Y-%m-%d")] = DayInCalendar(
+ d, d >= self.firstdate and d <= self.lastdate
+ )
def is_single_week(self):
return hasattr(self, "week")
-
def is_full_month(self):
return hasattr(self, "month")
-
def calendar_days_list(self):
return list(self.calendar_days.values())
-class CalendarMonth(CalendarList):
+class CalendarMonth(CalendarList):
def __init__(self, year, month, filter):
self.year = year
self.month = month
@@ -192,7 +198,6 @@ class CalendarMonth(CalendarList):
class CalendarWeek(CalendarList):
-
def __init__(self, year, week, filter):
self.year = year
self.week = week
@@ -210,9 +215,8 @@ class CalendarWeek(CalendarList):
class CalendarDay(CalendarList):
-
def __init__(self, date, filter=None):
super().__init__(date, date, filter, exact=True)
def get_events(self):
- return self.calendar_days_list()[0].events
\ No newline at end of file
+ return self.calendar_days_list()[0].events
diff --git a/src/agenda_culturel/celery.py b/src/agenda_culturel/celery.py
index 8c68cd4..b92b927 100644
--- a/src/agenda_culturel/celery.py
+++ b/src/agenda_culturel/celery.py
@@ -12,8 +12,6 @@ from .import_tasks.extractor_ical import *
from .import_tasks.custom_extractors import *
-
-
# Set the default Django settings module for the 'celery' program.
APP_ENV = os.getenv("APP_ENV", "dev")
os.environ.setdefault("DJANGO_SETTINGS_MODULE", f"agenda_culturel.settings.{APP_ENV}")
@@ -32,11 +30,14 @@ app.config_from_object("django.conf:settings", namespace="CELERY")
# Load task modules from all registered Django apps.
app.autodiscover_tasks()
+
def close_import_task(taskid, success, error_message, importer):
from agenda_culturel.models import BatchImportation
task = BatchImportation.objects.get(celery_id=taskid)
- task.status = BatchImportation.STATUS.SUCCESS if success else BatchImportation.STATUS.FAILED
+ task.status = (
+ BatchImportation.STATUS.SUCCESS if success else BatchImportation.STATUS.FAILED
+ )
task.nb_initial = importer.get_nb_events()
task.nb_imported = importer.get_nb_imported_events()
task.nb_updated = importer.get_nb_updated_events()
@@ -59,12 +60,11 @@ def import_events_from_json(self, json):
# save batch importation
importation.save()
-
logger.info("Import events from json: {}".format(self.request.id))
importer = DBImporterEvents(self.request.id)
- #try:
+ # try:
success, error_message = importer.import_events(json)
# finally, close task
@@ -82,7 +82,7 @@ def run_recurrent_import(self, pk):
logger.info("Run recurrent import: {}".format(self.request.id))
- # get the recurrent import
+ # get the recurrent import
rimport = RecurrentImport.objects.get(pk=pk)
# create a batch importation
@@ -94,7 +94,11 @@ def run_recurrent_import(self, pk):
importer = DBImporterEvents(self.request.id)
# prepare downloading and extracting processes
- downloader = SimpleDownloader() if rimport.downloader == RecurrentImport.DOWNLOADER.SIMPLE else ChromiumHeadlessDownloader()
+ downloader = (
+ SimpleDownloader()
+ if rimport.downloader == RecurrentImport.DOWNLOADER.SIMPLE
+ else ChromiumHeadlessDownloader()
+ )
if rimport.processor == RecurrentImport.PROCESSOR.ICAL:
extractor = ICALExtractor()
elif rimport.processor == RecurrentImport.PROCESSOR.ICALNOBUSY:
@@ -127,7 +131,12 @@ def run_recurrent_import(self, pk):
try:
# get events from website
- events = u2e.process(url, browsable_url, default_values = {"category": category, "location": location, "tags": tags}, published = published)
+ events = u2e.process(
+ url,
+ browsable_url,
+ default_values={"category": category, "location": location, "tags": tags},
+ published=published,
+ )
# convert it to json
json_events = json.dumps(events, default=str)
@@ -145,15 +154,20 @@ def run_recurrent_import(self, pk):
@app.task(bind=True)
def daily_imports(self):
from agenda_culturel.models import RecurrentImport
+
logger.info("Imports quotidiens")
- imports = RecurrentImport.objects.filter(recurrence=RecurrentImport.RECURRENCE.DAILY)
+ imports = RecurrentImport.objects.filter(
+ recurrence=RecurrentImport.RECURRENCE.DAILY
+ )
for imp in imports:
run_recurrent_import.delay(imp.pk)
+
@app.task(bind=True)
def run_all_recurrent_imports(self):
from agenda_culturel.models import RecurrentImport
+
logger.info("Imports complets")
imports = RecurrentImport.objects.all()
@@ -164,12 +178,16 @@ def run_all_recurrent_imports(self):
@app.task(bind=True)
def weekly_imports(self):
from agenda_culturel.models import RecurrentImport
+
logger.info("Imports hebdomadaires")
- imports = RecurrentImport.objects.filter(recurrence=RecurrentImport.RECURRENCE.WEEKLY)
+ imports = RecurrentImport.objects.filter(
+ recurrence=RecurrentImport.RECURRENCE.WEEKLY
+ )
for imp in imports:
run_recurrent_import.delay(imp.pk)
+
app.conf.beat_schedule = {
"daily_imports": {
"task": "agenda_culturel.celery.daily_imports",
@@ -179,9 +197,8 @@ app.conf.beat_schedule = {
"weekly_imports": {
"task": "agenda_culturel.celery.weekly_imports",
# Daily imports on Mondays at 2:22 a.m.
- "schedule": crontab(hour=2, minute=22, day_of_week='mon'),
+ "schedule": crontab(hour=2, minute=22, day_of_week="mon"),
},
}
app.conf.timezone = "Europe/Paris"
-
diff --git a/src/agenda_culturel/db_importer.py b/src/agenda_culturel/db_importer.py
index ae93035..6c513d4 100644
--- a/src/agenda_culturel/db_importer.py
+++ b/src/agenda_culturel/db_importer.py
@@ -4,11 +4,11 @@ from datetime import datetime
from django.utils import timezone
import logging
+
logger = logging.getLogger(__name__)
class DBImporterEvents:
-
def __init__(self, celery_id):
self.celery_id = celery_id
self.error_message = ""
@@ -60,7 +60,7 @@ class DBImporterEvents:
# get events
for event in structure["events"]:
# only process events if they are today or the days after
-
+
if self.event_takes_place_today_or_after(event):
# set a default "last modified date"
if "last_modified" not in event and self.date is not None:
@@ -69,7 +69,7 @@ class DBImporterEvents:
# conversion to Event, and return an error if it failed
if not self.load_event(event):
return (False, self.error_message)
-
+
# finally save the loaded events in database
self.save_imported()
@@ -92,27 +92,31 @@ class DBImporterEvents:
return event["end_day"] >= self.today
def save_imported(self):
- self.db_event_objects, self.nb_updated, self.nb_removed = Event.import_events(self.event_objects, remove_missing_from_source=self.url)
+ self.db_event_objects, self.nb_updated, self.nb_removed = Event.import_events(
+ self.event_objects, remove_missing_from_source=self.url
+ )
-
def is_valid_event_structure(self, event):
if "title" not in event:
- self.error_message = "JSON is not correctly structured: one event without title"
+ self.error_message = (
+ "JSON is not correctly structured: one event without title"
+ )
return False
if "start_day" not in event:
- self.error_message = "JSON is not correctly structured: one event without start_day"
+ self.error_message = (
+ "JSON is not correctly structured: one event without start_day"
+ )
return False
return True
-
def load_event(self, event):
if self.is_valid_event_structure(event):
- logger.warning("Valid event: {} {}".format(event["last_modified"], event["title"]))
+ logger.warning(
+ "Valid event: {} {}".format(event["last_modified"], event["title"])
+ )
event_obj = Event.from_structure(event, self.url)
self.event_objects.append(event_obj)
return True
else:
logger.warning("Not valid event: {}".format(event))
return False
-
-
diff --git a/src/agenda_culturel/forms.py b/src/agenda_culturel/forms.py
index d576147..9fd9f27 100644
--- a/src/agenda_culturel/forms.py
+++ b/src/agenda_culturel/forms.py
@@ -1,8 +1,31 @@
-from django.forms import ModelForm, ValidationError, TextInput, Form, URLField, MultipleHiddenInput, Textarea, CharField, ChoiceField, RadioSelect, MultipleChoiceField, BooleanField, HiddenInput, ModelChoiceField
+from django.forms import (
+ ModelForm,
+ ValidationError,
+ TextInput,
+ Form,
+ URLField,
+ MultipleHiddenInput,
+ Textarea,
+ CharField,
+ ChoiceField,
+ RadioSelect,
+ MultipleChoiceField,
+ BooleanField,
+ HiddenInput,
+ ModelChoiceField,
+)
from datetime import date
from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget
-from .models import Event, BatchImportation, RecurrentImport, CategorisationRule, ModerationAnswer, ModerationQuestion, Place
+from .models import (
+ Event,
+ BatchImportation,
+ RecurrentImport,
+ CategorisationRule,
+ ModerationAnswer,
+ ModerationQuestion,
+ Place,
+)
from django.utils.translation import gettext_lazy as _
from string import ascii_uppercase as auc
from .templatetags.utils_extra import int_to_abc
@@ -12,6 +35,7 @@ from django.utils.formats import localize
from .templatetags.event_extra import event_field_verbose_name, field_to_html
import logging
+
logger = logging.getLogger(__name__)
@@ -22,45 +46,63 @@ class EventSubmissionForm(Form):
class DynamicArrayWidgetURLs(DynamicArrayWidget):
template_name = "agenda_culturel/widgets/widget-urls.html"
+
class DynamicArrayWidgetTags(DynamicArrayWidget):
template_name = "agenda_culturel/widgets/widget-tags.html"
+
class RecurrentImportForm(ModelForm):
class Meta:
model = RecurrentImport
- fields = '__all__'
+ fields = "__all__"
widgets = {
- 'defaultTags': DynamicArrayWidgetTags(),
+ "defaultTags": DynamicArrayWidgetTags(),
}
+
class CategorisationRuleImportForm(ModelForm):
class Meta:
model = CategorisationRule
- fields = '__all__'
+ fields = "__all__"
+
class EventForm(ModelForm):
-
class Meta:
model = Event
- exclude = ["possibly_duplicated", "imported_date", "modified_date", "moderated_date"]
+ exclude = [
+ "possibly_duplicated",
+ "imported_date",
+ "modified_date",
+ "moderated_date",
+ ]
widgets = {
- 'start_day': TextInput(attrs={'type': 'date', 'onchange': 'update_datetimes(event);', "onfocus": "this.oldvalue = this.value;"}),
- 'start_time': TextInput(attrs={'type': 'time', 'onchange': 'update_datetimes(event);', "onfocus": "this.oldvalue = this.value;"}),
- 'end_day': TextInput(attrs={'type': 'date'}),
- 'end_time': TextInput(attrs={'type': 'time'}),
- 'uuids': MultipleHiddenInput(),
- 'import_sources': MultipleHiddenInput(),
- 'reference_urls': DynamicArrayWidgetURLs(),
- 'tags': DynamicArrayWidgetTags(),
+ "start_day": TextInput(
+ attrs={
+ "type": "date",
+ "onchange": "update_datetimes(event);",
+ "onfocus": "this.oldvalue = this.value;",
+ }
+ ),
+ "start_time": TextInput(
+ attrs={
+ "type": "time",
+ "onchange": "update_datetimes(event);",
+ "onfocus": "this.oldvalue = this.value;",
+ }
+ ),
+ "end_day": TextInput(attrs={"type": "date"}),
+ "end_time": TextInput(attrs={"type": "time"}),
+ "uuids": MultipleHiddenInput(),
+ "import_sources": MultipleHiddenInput(),
+ "reference_urls": DynamicArrayWidgetURLs(),
+ "tags": DynamicArrayWidgetTags(),
}
-
def __init__(self, *args, **kwargs):
- is_authenticated = kwargs.pop('is_authenticated', False)
+ is_authenticated = kwargs.pop("is_authenticated", False)
super().__init__(*args, **kwargs)
if not is_authenticated:
- del self.fields['status']
-
+ del self.fields["status"]
def clean_end_day(self):
start_day = self.cleaned_data.get("start_day")
@@ -82,40 +124,71 @@ class EventForm(ModelForm):
# both start and end time are defined
if start_time is not None and end_time is not None:
if start_time > end_time:
- raise ValidationError(_("The end time cannot be earlier than the start time."))
+ raise ValidationError(
+ _("The end time cannot be earlier than the start time.")
+ )
return end_time
-
class BatchImportationForm(Form):
- json = CharField(label="JSON", widget=Textarea(attrs={"rows":"10"}), help_text=_("JSON in the format expected for the import."), required=True)
+ json = CharField(
+ label="JSON",
+ widget=Textarea(attrs={"rows": "10"}),
+ help_text=_("JSON in the format expected for the import."),
+ required=True,
+ )
class FixDuplicates(Form):
-
-
action = ChoiceField()
def __init__(self, *args, **kwargs):
- nb_events = kwargs.pop('nb_events', None)
+ nb_events = kwargs.pop("nb_events", None)
super().__init__(*args, **kwargs)
if nb_events == 2:
choices = [("NotDuplicates", "Ces événements sont différents")]
- choices += [("SelectA", "Ces événements sont identiques, on garde A et on met B à la corbeille")]
- choices += [("SelectB", "Ces événements sont identiques, on garde B et on met A à la corbeille")]
- choices += [("Merge", "Ces événements sont identiques, on fusionne à la main")]
+ choices += [
+ (
+ "SelectA",
+ "Ces événements sont identiques, on garde A et on met B à la corbeille",
+ )
+ ]
+ choices += [
+ (
+ "SelectB",
+ "Ces événements sont identiques, on garde B et on met A à la corbeille",
+ )
+ ]
+ choices += [
+ ("Merge", "Ces événements sont identiques, on fusionne à la main")
+ ]
else:
choices = [("NotDuplicates", "Ces événements sont tous différents")]
for i in auc[0:nb_events]:
- choices += [("Remove" + i, "L'événement " + i + " n'est pas identique aux autres, on le rend indépendant")]
+ choices += [
+ (
+ "Remove" + i,
+ "L'événement "
+ + i
+ + " n'est pas identique aux autres, on le rend indépendant",
+ )
+ ]
for i in auc[0:nb_events]:
- choices += [("Select" + i, "Ces événements sont identiques, on garde " + i + " et on met les autres à la corbeille")]
- choices += [("Merge", "Ces événements sont identiques, on fusionne à la main")]
+ choices += [
+ (
+ "Select" + i,
+ "Ces événements sont identiques, on garde "
+ + i
+ + " et on met les autres à la corbeille",
+ )
+ ]
+ choices += [
+ ("Merge", "Ces événements sont identiques, on fusionne à la main")
+ ]
-
- self.fields['action'].choices = choices
+ self.fields["action"].choices = choices
def is_action_no_duplicates(self):
return self.cleaned_data["action"] == "NotDuplicates"
@@ -145,26 +218,28 @@ class FixDuplicates(Form):
class SelectEventInList(Form):
-
event = ChoiceField()
def __init__(self, *args, **kwargs):
- events = kwargs.pop('events', None)
+ events = kwargs.pop("events", None)
super().__init__(*args, **kwargs)
- self.fields['event'].choices = [(e.pk, str(e.start_day) + " " + e.title + ", " + e.location) for e in events]
+ self.fields["event"].choices = [
+ (e.pk, str(e.start_day) + " " + e.title + ", " + e.location) for e in events
+ ]
class MergeDuplicates(Form):
-
checkboxes_fields = ["reference_urls", "description"]
def __init__(self, *args, **kwargs):
- self.duplicates = kwargs.pop('duplicates', None)
+ self.duplicates = kwargs.pop("duplicates", None)
nb_events = self.duplicates.nb_duplicated()
super().__init__(*args, **kwargs)
- choices = [("event" + i, "Valeur de l'évenement " + i) for i in auc[0:nb_events]]
+ choices = [
+ ("event" + i, "Valeur de l'évenement " + i) for i in auc[0:nb_events]
+ ]
for f in self.duplicates.get_items_comparison():
if not f["similar"]:
@@ -172,42 +247,61 @@ class MergeDuplicates(Form):
self.fields[f["key"]] = MultipleChoiceField(choices=choices)
self.fields[f["key"]].initial = choices[0][0]
else:
- self.fields[f["key"]] = ChoiceField(widget=RadioSelect, choices=choices)
+ self.fields[f["key"]] = ChoiceField(
+ widget=RadioSelect, choices=choices
+ )
self.fields[f["key"]].initial = choices[0][0]
-
def as_grid(self):
result = '
'
for i, e in enumerate(self.duplicates.get_duplicated()):
result += '
'
- result += '
' + int_to_abc(i) + '
'
- result += '
'
- result += '- ' + e.title + '
'
- result += '- Création : ' + localize(localtime(e.created_date)) + '
'
- result += '- Dernière modification : ' + localize(localtime(e.modified_date)) + '
'
+ result += '' + int_to_abc(i) + "
"
+ result += ""
+ result += (
+ '- ' + e.title + "
"
+ )
+ result += (
+ "- Création : " + localize(localtime(e.created_date)) + "
"
+ )
+ result += (
+ "- Dernière modification : "
+ + localize(localtime(e.modified_date))
+ + "
"
+ )
if e.imported_date:
- result += '- Dernière importation : ' + localize(localtime(e.imported_date)) + '
'
- result += '
'
- result += '
'
- result += '
'
+ result += (
+ "Dernière importation : "
+ + localize(localtime(e.imported_date))
+ + ""
+ )
+ result += ""
+ result += ""
+ result += ""
for e in self.duplicates.get_items_comparison():
key = e["key"]
result += "" + event_field_verbose_name(e["key"]) + "
"
if e["similar"]:
- result += 'Identique :' + str(field_to_html(e["values"], e["key"])) + '
'
+ result += (
+ 'Identique :'
+ + str(field_to_html(e["values"], e["key"]))
+ + "
"
+ )
else:
- result += '