diff --git a/src/agenda_culturel/calendar.py b/src/agenda_culturel/calendar.py index a72f7c0..5848ca5 100644 --- a/src/agenda_culturel/calendar.py +++ b/src/agenda_culturel/calendar.py @@ -132,7 +132,7 @@ class CalendarList: self.events = qs.filter( (Q(recurrence_dtend__isnull=True) & Q(recurrence_dtstart__lte=lastdatetime)) | (Q(recurrence_dtend__isnull=False) & ~(Q(recurrence_dtstart__gt=lastdatetime) | Q(recurrence_dtend__lt=startdatetime))) - ).order_by("start_day", "start_time") + ).order_by("start_time") firstdate = datetime.fromordinal(self.c_firstdate.toordinal()) if firstdate.tzinfo is None or firstdate.tzinfo.utcoffset(firstdate) is None: @@ -211,5 +211,8 @@ class CalendarWeek(CalendarList): class CalendarDay(CalendarList): - def __init__(self, date, filter): + def __init__(self, date, filter=None): super().__init__(date, date, filter, exact=True) + + def get_events(self): + return self.calendar_days_list()[0].events \ No newline at end of file diff --git a/src/agenda_culturel/forms.py b/src/agenda_culturel/forms.py index 1d1becb..fd0accd 100644 --- a/src/agenda_culturel/forms.py +++ b/src/agenda_culturel/forms.py @@ -1,9 +1,19 @@ -from django.forms import ModelForm, ValidationError, TextInput, Form, URLField, MultipleHiddenInput, Textarea, CharField +from django.forms import ModelForm, ValidationError, TextInput, Form, URLField, MultipleHiddenInput, Textarea, CharField, ChoiceField, RadioSelect, MultipleChoiceField from datetime import date from django_better_admin_arrayfield.forms.widgets import DynamicArrayWidget from .models import Event, BatchImportation from django.utils.translation import gettext_lazy as _ +from string import ascii_uppercase as auc +from .templatetags.utils_extra import int_to_abc +from django.utils.safestring import mark_safe +from django.utils.timezone import localtime +from django.utils.formats import localize +from .templatetags.event_extra import event_field_verbose_name, field_to_html + +import logging +logger = logging.getLogger(__name__) + class EventSubmissionForm(Form): url = URLField(max_length=512) @@ -87,3 +97,146 @@ class BatchImportationForm(ModelForm): return cleaned_data +class FixDuplicates(Form): + + + action = ChoiceField() + + def __init__(self, *args, **kwargs): + nb_events = kwargs.pop('nb_events', None) + super().__init__(*args, **kwargs) + + if nb_events == 2: + choices = [("NotDuplicates", "Ces événements sont différents")] + choices += [("SelectA", "Ces événements sont identiques, on garde A et on met B à la corbeile")] + choices += [("SelectB", "Ces événements sont identiques, on garde B et on met A à la corbeille")] + choices += [("Merge", "Ces événements sont identiques, on fusionne à la main")] + else: + choices = [("NotDuplicates", "Ces événements sont tous différents")] + for i in auc[0:nb_events]: + choices += [("Remove" + i, "L'événement " + i + " n'est pas identique aux autres, on le rend indépendant")] + for i in auc[0:nb_events]: + choices += [("Select" + i, "Ces événements sont identiques, on garde " + i + " et on met les autres à la corbeille")] + choices += [("Merge", "Ces événements sont identiques, on fusionne à la main")] + + + self.fields['action'].choices = choices + + def is_action_no_duplicates(self): + return self.cleaned_data["action"] == "NotDuplicates" + + def is_action_select(self): + return self.cleaned_data["action"].startswith("Select") + + def is_action_remove(self): + return self.cleaned_data["action"].startswith("Remove") + + def get_selected_event_code(self): + if self.is_action_select() or self.is_action_remove(): + return self.cleaned_data["action"][-1] + else: + return None + + def get_selected_event_id(self): + selected = self.get_selected_event_code() + if selected is None: + return None + else: + return auc.rfind(selected) + + def get_selected_event(self, edup): + selected = self.get_selected_event_id() + return edup.get_duplicated()[selected] + + +class SelectEventInList(Form): + + event = ChoiceField() + + def __init__(self, *args, **kwargs): + events = kwargs.pop('events', None) + super().__init__(*args, **kwargs) + + self.fields['event'].choices = [(e.pk, str(e.start_day) + " " + e.title + ", " + e.location) for e in events] + + +class MergeDuplicates(Form): + + checkboxes_fields = ["reference_urls", "description"] + + def __init__(self, *args, **kwargs): + self.duplicates = kwargs.pop('duplicates', None) + nb_events = self.duplicates.nb_duplicated() + super().__init__(*args, **kwargs) + + choices = [("event" + i, "Valeur de l'évenement " + i) for i in auc[0:nb_events]] + + for f in self.duplicates.get_items_comparison(): + if not f["similar"]: + if f["key"] in MergeDuplicates.checkboxes_fields: + self.fields[f["key"]] = MultipleChoiceField(choices=choices) + self.fields[f["key"]].initial = choices[0][0] + else: + self.fields[f["key"]] = ChoiceField(widget=RadioSelect, choices=choices) + self.fields[f["key"]].initial = choices[0][0] + + + def as_grid(self): + result = '
' + for i, e in enumerate(self.duplicates.get_duplicated()): + result += '
' + result += '
' + int_to_abc(i) + '
' + result += '' + result += '
' + result += '
' + + for e in self.duplicates.get_items_comparison(): + key = e["key"] + result += "

" + event_field_verbose_name(e["key"]) + "

" + if e["similar"]: + result += '
Identique :' + str(field_to_html(e["values"], e["key"])) + '
' + else: + result += '
' + result += '
' + if hasattr(self, "cleaned_data"): + checked = self.cleaned_data.get(key) + else: + checked = self.fields[key].initial + + for i, (v, radio) in enumerate(zip(e["values"], self.fields[e["key"]].choices)): + result += '
' + id = 'id_' + key + '_' + str(i) + value = 'event' + auc[i] + + result += '' + str(field_to_html(v, e["key"])) + '
' + result += "
" + + return mark_safe(result) + + + def get_selected_events_id(self, key): + value = self.cleaned_data.get(key) + if not key in self.fields: + return None + else: + if isinstance(value, list): + return [auc.rfind(v[-1]) for v in value] + else: + return auc.rfind(value[-1]) diff --git a/src/agenda_culturel/models.py b/src/agenda_culturel/models.py index e920bea..f7eb095 100644 --- a/src/agenda_culturel/models.py +++ b/src/agenda_culturel/models.py @@ -134,7 +134,8 @@ class DuplicatedEvents(models.Model): def get_item_comparion(self, attr): values = [getattr(e, attr) for e in self.get_duplicated()] - if isinstance(values[0], list): + + if len([x for x in [isinstance(i, list) for i in values] if x is True]) > 0: hashable_values = "; ".join([str(v) for v in values]) else: hashable_values = values @@ -144,7 +145,7 @@ class DuplicatedEvents(models.Model): return { "similar": False, "key": attr, "values": values } def get_items_comparison(self): - return [self.get_item_comparion(e) for e in Event.data_fields() + ["local_image"]] + return [self.get_item_comparion(e) for e in Event.data_fields(all=True)] class Event(models.Model): @@ -274,6 +275,11 @@ class Event(models.Model): # if the download is ok, then create the corresponding file object self.local_image = File(name=basename, file=open(tmpfile, "rb")) + def set_skip_duplicate_check(self): + self.skip_duplicate_check = True + + def is_skip_duplicate_check(self): + return hasattr(self, "skip_duplicate_check") def is_in_importation_process(self): return hasattr(self, "in_importation_process") @@ -302,7 +308,7 @@ class Event(models.Model): # return a copy of the current object for each recurrence between first an last date (included) def get_recurrences_between(self, firstdate, lastdate): - if self.recurrences is None: + if not self.has_recurrences(): return [self] else: result = [] @@ -330,8 +336,7 @@ class Event(models.Model): etime = time.fromisoformat(self.end_time) if isinstance(self.end_time, str) else time() if self.end_time is None else self.end_time self.recurrence_dtstart = datetime.combine(sday, stime) - # TODO: see https://forge.chapril.org/jmtrivial/agenda_culturel/issues/65 - if self.recurrences is None or len(self.recurrences.rrules) == 0: + if not self.has_recurrences(): if self.end_day is None: self.dtend = None else: @@ -353,8 +358,6 @@ class Event(models.Model): def prepare_save(self): self.update_modification_dates() - # TODO: update recurrences.dtstart et recurrences.dtend - self.update_recurrence_dtstartend() # if the image is defined but not locally downloaded @@ -366,8 +369,8 @@ class Event(models.Model): self.prepare_save() - # check for similar events if no duplicated is known - if self.possibly_duplicated is None: + # check for similar events if no duplicated is known only if the event is created + if self.pk is None and self.possibly_duplicated is None and not self.is_skip_duplicate_check(): # and if this is not an importation process if not self.is_in_importation_process(): similar_events = self.find_similar_events() @@ -376,8 +379,9 @@ class Event(models.Model): if len(similar_events) != 0: self.set_possibly_duplicated(similar_events) - elif self.possibly_duplicated is not None and self.possibly_duplicated.nb_duplicated() == 1: - # delete duplicated group if it's only with one element + + # delete duplicated group if it's only with one element + if self.possibly_duplicated is not None and self.possibly_duplicated.nb_duplicated() == 1: self.possibly_duplicated.delete() self.possibly_duplicated = None @@ -386,8 +390,6 @@ class Event(models.Model): def from_structure(event_structure, import_source = None): - if event_structure["title"].endswith("ole"): - logger.warning("on choope {}".format(event_structure)) if "category" in event_structure and event_structure["category"] is not None: event_structure["category"] = Category.objects.get(name=event_structure["category"]) @@ -480,32 +482,45 @@ class Event(models.Model): def set_possibly_duplicated(self, events): + # get existing groups groups = list(set([e.possibly_duplicated for e in events] + [self.possibly_duplicated])) groups = [g for g in groups if g is not None] + # do we have to create a new group? if len(groups) == 0: group = DuplicatedEvents.objects.create() - logger.warning("set possibly duplicated 0 {}".format(group)) else: # otherwise merge existing groups group = DuplicatedEvents.merge_groups(groups) - logger.warning("set possibly duplicated not 0 {}".format(group)) group.save() + # set the possibly duplicated group for the current object self.possibly_duplicated = group # and for the other events for e in events: e.possibly_duplicated = group - # finally save the other events - Event.objects.bulk_update(events, fields=["possibly_duplicated"]) + + # finally update all events (including current) + Event.objects.bulk_update(events + [self], fields=["possibly_duplicated"]) - def data_fields(): - return ["title", "location", "start_day", "start_time", "end_day", "end_time", "description", "image", "image_alt", "reference_urls", "recurrences"] + def data_fields(all=False): + if all: + result = ["category"] + else: + result = [] + + result += ["title", "location", "start_day", "start_time", "end_day", "end_time", "description", "image"] + if all: + result += ["local_image"] + result += ["image_alt", "reference_urls", "recurrences"] + if all: + result += ["tags"] + return result def same_event_by_data(self, other): for attr in Event.data_fields(): diff --git a/src/agenda_culturel/templates/agenda_culturel/duplicates.html b/src/agenda_culturel/templates/agenda_culturel/duplicates.html index 4ae958f..7582450 100644 --- a/src/agenda_culturel/templates/agenda_culturel/duplicates.html +++ b/src/agenda_culturel/templates/agenda_culturel/duplicates.html @@ -20,7 +20,7 @@ {% for obj in paginator_filter %} {% with obj.get_duplicated as events %}
-
Possible duplication : {{ events|length }} événements le {{ events.0.start_day }} +
Possible duplication : {{ events|length }} événements le {{ events.0.start_day }}
{% for e in object.get_duplicated %}
- {{ forloop.counter }} + {{ forloop.counter0|int_to_abc }}
    +
  • {{ e.title }}
  • Création : {{ e.created_date }}
  • Dernière modification : {{ e.modified_date }}
  • {% if e.imported_date %}
  • Dernière importation : {{ e.imported_date }}
  • {% endif %} @@ -38,7 +60,7 @@ {% else %}
    {% for i in e.values %} -
    {{ forloop.counter }}
    {% field_to_html i e.key %}
    +
    {{ forloop.counter0|int_to_abc }}
    {% field_to_html i e.key %}
    {% endfor %}
    {% endif %} diff --git a/src/agenda_culturel/templates/agenda_culturel/merge_duplicate.html b/src/agenda_culturel/templates/agenda_culturel/merge_duplicate.html new file mode 100644 index 0000000..aeb7c33 --- /dev/null +++ b/src/agenda_culturel/templates/agenda_culturel/merge_duplicate.html @@ -0,0 +1,31 @@ +{% extends "agenda_culturel/page.html" %} + +{% load utils_extra %} +{% load event_extra %} + +{% block title %}Fusionner les événements dupliqués{% endblock %} + +{% load cat_extra %} +{% block entete_header %} + {% css_categories %} +{% endblock %} + +{% block content %} +
    +
    +

    Fusionner les événements dupliqués

    +

    Pour chacun des champs non identiques, choisissez la version qui vous convient pour créer un événement + résultat de la fusion. Les événements source seront déplacés dans la corbeille.

    + +
    +
    + {% csrf_token %} + {{ form.as_grid }} +
    + Annuler + +
    +
    +
    +{% endblock %} + diff --git a/src/agenda_culturel/templates/agenda_culturel/page-event.html b/src/agenda_culturel/templates/agenda_culturel/page-event.html index d1ae9d4..0c3699b 100644 --- a/src/agenda_culturel/templates/agenda_culturel/page-event.html +++ b/src/agenda_culturel/templates/agenda_culturel/page-event.html @@ -77,6 +77,13 @@ {% endif %}
{% endif %} +
+ {% if user.is_authenticated %} + Marquer comme doublon + {% else %} + Signaler comme doublon + {% endif %} +
diff --git a/src/agenda_culturel/templates/agenda_culturel/set_duplicate.html b/src/agenda_culturel/templates/agenda_culturel/set_duplicate.html new file mode 100644 index 0000000..4667206 --- /dev/null +++ b/src/agenda_culturel/templates/agenda_culturel/set_duplicate.html @@ -0,0 +1,30 @@ +{% extends "agenda_culturel/page.html" %} + +{% load utils_extra %} +{% load event_extra %} + +{% block title %}{% if user.is_authenticated %}Marquer comme doublon{% else %}Signaler comme doublon{% endif %}{% endblock %} + +{% load cat_extra %} +{% block entete_header %} + {% css_categories %} +{% endblock %} + +{% block content %} +
+
+

{% if user.is_authenticated %}Marquer comme doublon{% else %}Signaler comme doublon{% endif %}

+

De quel événement listé ci-dessous l'événement sélectionné est-il un doublon ?

+ +
{% csrf_token %} + {{ form.as_p }} +
+ Annuler + +
+
+
+ {% include "agenda_culturel/single-event/event-single-inc.html" with event=event %} +
+{% endblock %} + diff --git a/src/agenda_culturel/templatetags/utils_extra.py b/src/agenda_culturel/templatetags/utils_extra.py index 20b118d..1e79e86 100644 --- a/src/agenda_culturel/templatetags/utils_extra.py +++ b/src/agenda_culturel/templatetags/utils_extra.py @@ -5,6 +5,7 @@ from urllib.parse import urlparse from datetime import timedelta, date from django.urls import reverse_lazy from django.templatetags.static import static +from string import ascii_uppercase as auc register = template.Library() @@ -66,3 +67,7 @@ def picto_from_name(name, datatooltip=""): result = '' + result + '' return mark_safe(result) + +@register.filter +def int_to_abc(d): + return auc[int(d)] diff --git a/src/agenda_culturel/urls.py b/src/agenda_culturel/urls.py index 6840157..77f6490 100644 --- a/src/agenda_culturel/urls.py +++ b/src/agenda_culturel/urls.py @@ -24,6 +24,7 @@ urlpatterns = [ path("event//edit", EventUpdateView.as_view(), name="edit_event"), path("event//change-status/", change_status_event, name="change_status_event"), path("event//delete", EventDeleteView.as_view(), name="delete_event"), + path("event/////set_duplicate", set_duplicate, name="set_duplicate"), path("ajouter", import_from_url, name="add_event"), path("admin/", admin.site.urls), path('accounts/', include('django.contrib.auth.urls')), @@ -40,7 +41,9 @@ urlpatterns = [ path("imports/add", BatchImportationCreateView.as_view(), name="add_import"), path("imports//cancel", cancel_import, name="cancel_import"), path("duplicates/", duplicates, name="duplicates"), - path("duplicates/", DuplicatedEventsDetailView.as_view(), name="fix_duplicate"), + path("duplicates/", DuplicatedEventsDetailView.as_view(), name="view_duplicate"), + path("duplicates//fix", fix_duplicate, name="fix_duplicate"), + path("duplicates//merge", merge_duplicate, name="merge_duplicate"), ] if settings.DEBUG: diff --git a/src/agenda_culturel/views.py b/src/agenda_culturel/views.py index 9bad6a7..c7b72f8 100644 --- a/src/agenda_culturel/views.py +++ b/src/agenda_culturel/views.py @@ -11,7 +11,7 @@ from django.http import HttpResponseRedirect from django.urls import reverse import urllib -from .forms import EventSubmissionForm, EventForm, BatchImportationForm +from .forms import EventSubmissionForm, EventForm, BatchImportationForm, FixDuplicates, SelectEventInList, MergeDuplicates from .models import Event, Category, StaticContent, ContactMessage, BatchImportation, DuplicatedEvents from django.utils import timezone @@ -35,6 +35,9 @@ from .extractors import ExtractorAllURLs from .celery import app as celery_app, import_events_from_json, import_events_from_url import unicodedata +import logging +logger = logging.getLogger(__name__) + def get_event_qs(request): @@ -154,7 +157,7 @@ def day_view(request, year = None, month = None, day = None): filter = EventFilter(request.GET, get_event_qs(request), request=request) cday = CalendarDay(day, filter) - context = {"day": day, "events": cday.calendar_days_list()[0].events, "filter": filter} + context = {"day": day, "events": cday.get_events(), "filter": filter} return render(request, 'agenda_culturel/page-day.html', context) @@ -538,6 +541,122 @@ class DuplicatedEventsDetailView(LoginRequiredMixin, DetailView): template_name = "agenda_culturel/fix_duplicate.html" +@login_required(login_url="/accounts/login/") +def merge_duplicate(request, pk): + edup = get_object_or_404(DuplicatedEvents, pk=pk) + form = MergeDuplicates(duplicates=edup) + + if request.method == 'POST': + form = MergeDuplicates(request.POST, duplicates=edup) + if form.is_valid(): + events = edup.get_duplicated() + + # build fields for the new event + new_event_data = {} + for f in edup.get_items_comparison(): + if f["similar"]: + new_event_data[f["key"]] = getattr(events[0], f["key"]) + else: + selected = form.get_selected_events_id(f["key"]) + if selected is None: + new_event_data[f["key"]] = None + elif isinstance(selected, list): + values = [x for x in [getattr(events[s], f["key"]) for s in selected] if x is not None] + if len(values) == 0: + new_event_data[f["key"]] = None + else: + if isinstance(values[0], str): + new_event_data[f["key"]] = "\n".join(values) + else: + new_event_data[f["key"]] = sum(values, []) + else: + new_event_data[f["key"]] = getattr(events[selected], f["key"]) + + for specific_tag in ["uuids", "import_sources"]: + new_event_data[specific_tag] = sum([x for x in [getattr(e, specific_tag) for e in events] if x is not None], []) + + # create a new event that merge the selected events + new_event = Event(**new_event_data) + new_event.set_skip_duplicate_check() + new_event.save() + + # move the old ones in trash + for e in events: + e.status = Event.STATUS.TRASH + Event.objects.bulk_update(events, fields=["status"]) + + messages.info(request, _("La fusion a été réalisée avec succès.")) + return HttpResponseRedirect(new_event.get_absolute_url()) + + return render(request, 'agenda_culturel/merge_duplicate.html', context={'form': form, 'object': edup}) + + +@login_required(login_url="/accounts/login/") +def fix_duplicate(request, pk): + + edup = get_object_or_404(DuplicatedEvents, pk=pk) + form = FixDuplicates(nb_events=edup.nb_duplicated()) + + if request.method == 'POST': + form = FixDuplicates(request.POST, nb_events=edup.nb_duplicated()) + + + if form.is_valid(): + if form.is_action_no_duplicates(): + events = edup.get_duplicated() + if len(events) == 0: + date = None + else: + s_events = [e for e in events if not e.has_recurrences()] + if len(s_events) != 0: + s_event = s_events[0] + else: + s_event = events[0] + date = s_event.start_day + + messages.success(request, _("Les événements ont été marqués comme non dupliqués.")) + edup.delete() + if date is None: + return HttpResponseRedirect(reverse_lazy("home")) + else: + return HttpResponseRedirect(reverse_lazy("day_view", args=[date.year, date.month, date.day])) + + elif form.is_action_select(): + selected = form.get_selected_event(edup) + not_selected = [e for e in edup.get_duplicated() if e != selected] + nb = len(not_selected) + for e in not_selected: + e.status = Event.STATUS.TRASH + Event.objects.bulk_update(not_selected, fields=["status"]) + url = selected.get_absolute_url() + edup.delete() + if nb == 1: + messages.success(request, _("L'événement sélectionné a été conservé, l'autre a été déplacé dans la corbeille.")) + else: + messages.success(request, _("L'événement sélectionné a été conservé, les autres ont été déplacés dans la corbeille.")) + return HttpResponseRedirect(url) + elif form.is_action_remove(): + event = form.get_selected_event(edup) + event.possibly_duplicated = None + event.save() + messages.success(request, _("L'événement a été retiré du groupe et rendu indépendant.")) + if edup.nb_duplicated() == 1: + return HttpResponseRedirect(event.get_absolute_url()) + else: + form = FixDuplicates(nb_events=edup.nb_duplicated()) + else: + return HttpResponseRedirect(reverse_lazy("merge_duplicate", args=[edup.pk])) + + return render(request, 'agenda_culturel/fix_duplicate.html', context={'form': form, 'object': edup}) + + + +class DuplicatedEventsUpdateView(LoginRequiredMixin, UpdateView): + model = DuplicatedEvents + fields = () + template_name = "agenda_culturel/fix_duplicate.html" + + @login_required(login_url="/accounts/login/") def duplicates(request): paginator = Paginator(DuplicatedEvents.objects.all(), 10) @@ -551,3 +670,28 @@ def duplicates(request): response = paginator.page(paginator.num_pages) return render(request, 'agenda_culturel/duplicates.html', {'filter': filter, 'paginator_filter': response} ) + +def set_duplicate(request, year, month, day, pk): + event = get_object_or_404(Event, pk=pk) + cday = CalendarDay(date(year, month, day)) + others = [e for e in cday.get_events() if e != event and (event.possibly_duplicated is None or event.possibly_duplicated != e.possibly_duplicated)] + + form = SelectEventInList(events=others) + + if request.method == 'POST': + form = SelectEventInList(request.POST, events=others) + if form.is_valid(): + selected = [o for o in others if o.pk == int(form.cleaned_data["event"])] + event.set_possibly_duplicated(selected) + event.save() + if request.user.is_authenticated: + messages.success(request, _("L'événement a été marqué dupliqué avec succès.")) + return HttpResponseRedirect(reverse_lazy("view_duplicate", args=[event.possibly_duplicated.pk])) + else: + messages.info(request, _("L'événement a été signalé comme dupliqué avec succès. Votre suggestion sera prochainement prise en charge par l'équipe de modération.")) + return HttpResponseRedirect(event.get_absolute_url()) + + + return render(request, 'agenda_culturel/set_duplicate.html', context={'form': form, 'event': event}) + +