from django.db import models, connection
from django.core.exceptions import FieldDoesNotExist
from django_better_admin_arrayfield.models.fields import ArrayField
from django.utils.translation import gettext_lazy as _
from django.utils.safestring import mark_safe
from django.template.defaultfilters import slugify
from django.utils.dateparse import parse_date
from django.urls import reverse
from colorfield.fields import ColorField
from django_ckeditor_5.fields import CKEditor5Field
from urllib.parse import urlparse
from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.contrib.auth.models import User, AnonymousUser
import emoji
from django.core.files.storage import default_storage
from django.contrib.sites.models import Site
from django.core.mail import send_mail
from django.template.loader import render_to_string
from autoslug import AutoSlugField
from django_resized import ResizedImageField

import uuid

import hashlib
import urllib.request
import os
from django.core.files import File
from django.utils import timezone
from django.contrib.postgres.search import TrigramSimilarity
from django.db.models import Q, Count, F, Subquery, OuterRef, Func
from django.db.models.functions import Lower
from django.contrib.postgres.lookups import Unaccent
import recurrence.fields
import recurrence
import copy
import unicodedata
from collections import defaultdict
from .import_tasks.generic_extractors.fbevent import CExtractor as FacebookEventExtractor
from .import_tasks.extractor import Extractor

from django.template.defaultfilters import date as _date
from datetime import time, timedelta, date
from django.utils.timezone import datetime
from django.utils import timezone

from location_field.models.spatial import LocationField
from django.contrib.gis.geos import Point

from .calendar import CalendarList, CalendarDay
from icalendar import Calendar as icalCal
from icalendar import Event as icalEvent

import logging

logger = logging.getLogger(__name__)


def remove_accents(input_str):
    if input_str is None:
        return None
    nfkd_form = unicodedata.normalize("NFKD", input_str)
    return "".join([c for c in nfkd_form if not unicodedata.combining(c)])


class StaticContent(models.Model):
    name = models.CharField(
        verbose_name=_("Name"),
        help_text=_("Category name"),
        max_length=512,
        unique=True,
    )
    text = CKEditor5Field(
        verbose_name=_("Content"), help_text=_("Text as shown to the visitors"),
        blank=True
    )
    url_path = models.CharField(
        verbose_name=_("URL path"),
        help_text=_("URL path where the content is included."),
    )

    class Meta:
        verbose_name = _("Static content")
        verbose_name_plural = _("Static contents")
        indexes = [
            models.Index(fields=['name']),
        ]

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return self.url_path


class Category(models.Model):
    default_name = "Sans catégorie"
    default_css_class = "cat-nocat"
    default_color = "#aaaaaa"

    COLOR_PALETTE = [
        ("#ea5545", "color 1"),
        ("#f46a9b", "color 2"),
        ("#ef9b20", "color 3"),
        ("#edbf33", "color 4"),
        ("#ede15b", "color 5"),
        ("#bdcf32", "color 6"),
        ("#87bc45", "color 7"),
        ("#27aeef", "color 8"),
        ("#b33dc6", "color 9"),
    ]

    name = models.CharField(
        verbose_name=_("Name"), help_text=_("Category name"), max_length=512
    )

    slug = AutoSlugField(null=True, default=None, unique=True, populate_from='name')

    color = ColorField(
        verbose_name=_("Color"),
        help_text=_("Color used as background for the category"),
        blank=True,
        null=True,
    )

    pictogram = models.FileField(
        verbose_name=_("Pictogram"),
        help_text=_("Pictogram of the category (svg format)"),
        max_length=1024,
        blank=True,
        null=True,
    )

    position = models.IntegerField(
        verbose_name=_("Position for ordering categories"), default=0
    )


    def save(self, *args, **kwargs):
        if self.color is None:
            existing_colors = [c.color for c in Category.objects.all()]
            if len(existing_colors) > len(Category.COLOR_PALETTE):
                self.color = "#CCCCCC"
            else:
                for c, n in Category.COLOR_PALETTE:
                    if c not in existing_colors:
                        self.color = c
                        break
            if self.color is None:
                self.color = "#CCCCCC"

        super(Category, self).save(*args, **kwargs)

    def get_default_category():
        try:
            # try to get an existing category
            default = Category.objects.get(name=Category.default_name)

            return default
        except:
            # if it does not exist, return it
            default, created = Category.objects.get_or_create(
                name=Category.default_name,
                color=Category.default_color,
            )
            return default

    def get_default_category_id():
        cat = Category.get_default_category()
        if cat:
            return cat.id
        else:
            return None

    def css_class(self):
        return "cat-" + str(self.id)

    def get_absolute_url(self):
        return reverse('home_category', kwargs={"cat": self.slug})

    def __str__(self):
        return self.name

    class Meta:
        verbose_name = _("Category")
        verbose_name_plural = _("Categories")
        indexes = [
            models.Index(fields=['name']),
        ]


class Tag(models.Model):

    name = models.CharField(
        verbose_name=_("Name"), help_text=_("Tag name"), max_length=512,
        unique=True
    )

    description = CKEditor5Field(
        verbose_name=_("Description"),
        help_text=_("Description of the tag"),
        blank=True,
        null=True,
    )

    principal = models.BooleanField(
        verbose_name=_("Principal"),
        help_text=_("This tag is highlighted as a main tag for visitors, particularly in the filter."),
        default=False,
    )

    in_excluded_suggestions =  models.BooleanField(
        verbose_name=_("In excluded suggestions"),
        help_text=_("This tag will be part of the excluded suggestions."),
        default=False,
    )

    in_included_suggestions =  models.BooleanField(
        verbose_name=_("In included suggestions"),
        help_text=_("This tag will be part of the included suggestions."),
        default=False,
    )

    class Meta:
        verbose_name = _("Tag")
        verbose_name_plural = _("Tags")
        indexes = [
            models.Index(fields=['name']),
        ]


    def get_absolute_url(self):
        return reverse("view_tag", kwargs={"t": self.name})


    def clear_cache():
        for exclude in [False, True]:
            for include in [False, True]:
                for nb_suggestions in [10]:
                    id_cache = 'all_tags ' + str(exclude) + ' ' + str(include) + ' ' + str(nb_suggestions)
                    id_cache = hashlib.md5(id_cache.encode("utf8")).hexdigest()
                    cache.delete(id_cache)

    def get_tag_groups(nb_suggestions=10, exclude=False, include=False, all=False):
        id_cache = 'all_tags ' + str(exclude) + ' ' + str(include) + ' ' + str(nb_suggestions)
        id_cache = hashlib.md5(id_cache.encode("utf8")).hexdigest()
        result = cache.get(id_cache)

        if not result:

            free_tags = Event.get_all_tags(False)
            f_tags = [t["tag"] for t in free_tags]

            obj_tags = Tag.objects

            if all:
                obj_tags = obj_tags.filter(Q(in_excluded_suggestions=True)|Q(in_included_suggestions=True)|Q(principal=True))
            else:
                if exclude:
                    obj_tags = obj_tags.filter(Q(in_excluded_suggestions=True))
                if include:
                    obj_tags = obj_tags.filter(Q(in_included_suggestions=True)|Q(principal=True))
                
                if not exclude and not include:
                    obj_tags = obj_tags.filter(principal=True)

            obj_tags = obj_tags.values_list("name", flat=True)

            if len(obj_tags) > nb_suggestions:
                nb_suggestions = len(obj_tags)
            
            tags = [{"tag": t["tag"], "count": 1000000 if t["tag"] in obj_tags else t["count"]} for t in free_tags]
            tags += [{"tag": o, "count": 0} for o in Tag.objects.filter(~Q(name__in=f_tags)).values_list("name", flat=True)]


            tags.sort(key=lambda x: -x["count"])

            tags1 = tags[0:nb_suggestions]
            tags1.sort(key=lambda x: emoji.demojize(remove_accents(x["tag"]).lower(), delimiters=('000', '')))
            tags2 = tags[nb_suggestions:]
            tags2.sort(key=lambda x: emoji.demojize(remove_accents(x["tag"]).lower(), delimiters=('000', '')))

            result = ((_('Suggestions'), [(t["tag"], t["tag"]) for t in tags1]),
                    (_('Others'), [(t["tag"], t["tag"]) for t in tags2]))
            
            cache.set(id_cache, result, 3000) # 50mn
        return result

    def __str__(self):
        return self.name



class DuplicatedEvents(models.Model):


    representative = models.ForeignKey(
        "Event",
        verbose_name=_("Representative event"),
        help_text=_("This event is the representative event of the duplicated events group"),
        null=True,
        default=None,
        on_delete=models.SET_DEFAULT,
    )

    class Meta:
        verbose_name = _("Duplicated events")
        verbose_name_plural = _("Duplicated events")
        indexes = [
            models.Index(fields=['representative']),
        ]


    def __init__(self, *args, **kwargs):
        self.events = None
        super().__init__(*args, **kwargs)

    def nb_duplicated(self):
        return self.event_set.count()

    def get_duplicated(self):
        if self.events is None:
            self.events = self.event_set.order_by("created_date").all()
        return self.events

    def get_absolute_url(self):
        return reverse("view_duplicate", kwargs={"pk": self.pk})

    def get_one_event(self):
        return self.representative

    def fixed(self):
        return not self.representative is None

    def is_published(self):
        return len([e for e in self.get_duplicated() if e.is_published()]) > 0

    def has_modified(self):
        return len([e for e in self.get_duplicated() if e.modified()]) > 0

    def has_local_version(self):
        return len([e for e in self.get_duplicated() if e.local_version()]) > 0

    def get_local_version(self):
        if self.representative and self.representative.local_version():
            return self.representative

        l = [e for e in self.get_duplicated() if e.local_version()]
        if len(l) == 0:
            return None
        else:
            l.sort(key=lambda x: x.modified_date, reverse=True)
            return l[0]

    def merge_into(self, other):
        # for all objects associated to this group
        for e in self.get_duplicated():
            # change their group membership
            e.other_versions = other
            # save them without updating modified date
            e.set_no_modification_date_changed()
            e.save()
        other.representative = None
        other.save()
        # then delete the empty group
        self.delete()

    # this method fixes the duplicated events by using the given event
    # as the representative one. 
    # if no event is given, the last one (by creation date) is selected.
    def fix(self, event=None):
        events = self.get_duplicated()
        if event is None:
            events = events.order_by("-created_date")
        for e in events:
            if event is None:
                event = e
        if not event is None:
            event.status = Event.STATUS.PUBLISHED
            self.representative = event
        Event.objects.bulk_update(events, fields=["status"])
        self.save()
        return len(events)
        

    def merge_groups(groups):
        if len(groups) == 0:
            return None
        elif len(groups) == 1:
            return groups[0]
        else:
            result = groups[0]
            for g in groups[1:]:
                g.merge_into(result)
            return result

    def get_items_comparison(self):
        return Event.get_comparison(self.get_duplicated())

    def remove_singletons():
        singletons = DuplicatedEvents.objects.annotate(nb_events=Count("event")).filter(
            nb_events__lte=1
        )
        nb, d = singletons.delete()
        return nb


    def not_fixed_qs(qs=None, fixed=False):
        if not qs:
            qs = DuplicatedEvents.objects
        
        qs = qs.annotate(nb_no_trash=Count("event", filter=~Q(event__status=Event.STATUS.TRASH)))
        q = ~Q(representative__isnull=True)|Q(nb_no_trash__lte=1)
        if fixed:
            return qs.filter(q)
        else:
            return qs.exclude(q)

    def save(self, *args, **kwargs):
        if self.representative and not self.representative in self.event_set.all():
            self.representative = None

        super().save(*args, **kwargs)

    def get_import_messages(self):
        msgs = []
        for e in self.get_duplicated():
            for m in e.message_set.filter(message_type__in=[Message.TYPE.IMPORT_PROCESS, Message.TYPE.UPDATE_PROCESS]).order_by("date"):
                msgs.append(m)
        return msgs

    def get_event_index(self, event):
        for i, e in enumerate(self.get_duplicated()):
            if e.pk == event.pk:
                return i
        return -1


class ReferenceLocation(models.Model):

    name = models.CharField(verbose_name=_("Name"), help_text=_("Name of the location"), unique=True, null=False)
    location = LocationField(based_fields=["name"], zoom=12, default=Point(3.08333, 45.783329), srid=4326)
    main = models.IntegerField(
        verbose_name=_("Main"),
        help_text=_("This location is one of the main locations (shown first higher values)."),
        default=0,
    )
    suggested_distance = models.IntegerField(
        verbose_name=_("Suggested distance (km)"),
        help_text=_("If this distance is given, this location is part of the suggested filters."),
        null=True,
        default=None
    )

    slug = AutoSlugField(null=True, default=None, unique=True, populate_from='name')

    class Meta:
        verbose_name = _("Reference location")
        verbose_name_plural = _("Reference locations")
        indexes = [
            models.Index(fields=['name']),
        ]

    def __str__(self):
        return self.name




class Place(models.Model):
    name = models.CharField(verbose_name=_("Name"), help_text=_("Name of the place"))
    address = models.CharField(
        verbose_name=_("Address"),
        help_text=_("Address of this place (without city name)"),
        blank=True,
        null=True,
    )
    postcode = models.CharField(verbose_name=_("Postcode"), help_text=_("The post code is not displayed, but makes it easier to find an address when you enter it."), blank=True, null=True)
    city = models.CharField(verbose_name=_("City"), help_text=_("City name"))
    location = LocationField(based_fields=["name", "address", "postcode", "city"], zoom=12, default=Point(3.08333, 45.783329))

    description = CKEditor5Field(
        verbose_name=_("Description"),
        help_text=_("Description of the place, including accessibility."),
        blank=True,
        null=True,
    )

    aliases = ArrayField(
        models.CharField(max_length=512),
        verbose_name=_("Alternative names"),
        help_text=_(
            "Alternative names or addresses used to match a place with the free-form location of an event."
        ),
        blank=True,
        null=True,
    )

    class Meta:
        verbose_name = _("Place")
        verbose_name_plural = _("Places")
        ordering = ["name"]
        indexes = [
            models.Index(fields=['name']),
            models.Index(fields=['city']),
            models.Index(fields=['location']),
        ]

    def __str__(self):
        if self.address:
            return self.name + ", " + self.address + ", " + self.city
        else:
            return self.name + ", " + self.city

    def get_absolute_url(self):
        return reverse("view_place_fullname", kwargs={"pk": self.pk, "extra": slugify(self.name)})

    def nb_events(self):
        return Event.objects.filter(exact_location=self).count()

    def nb_events_future(self):
        return Event.objects.filter(start_day__gte=datetime.now()).filter(exact_location=self).count()

    def match(self, event):
        if self.aliases and event.location:
            return event.location.strip() in self.aliases
        else:
            return False

    def associate_matching_events(self):
        u_events = Event.objects.filter(exact_location__isnull=True)

        to_be_updated = []
        # try to find matches
        for ue in u_events:
            if self.match(ue):
                ue.exact_location = self
                to_be_updated.append(ue)
                continue
        # update events with a location
        Event.objects.bulk_update(to_be_updated, fields=["exact_location"])
        return len(to_be_updated)

    def get_all_cities():
        try:
            tags = list(
                [
                    p["city"]
                    for p in Place.objects.values("city").distinct().order_by("city")
                ]
            )
        except:
            tags = []
        return tags

class Organisation(models.Model):
    name = models.CharField(
        verbose_name=_("Name"), help_text=_("Organisation name"), max_length=512, null=False, unique=True
    )    

    website = models.URLField(
        verbose_name=_("Website"),
        help_text=_("Website of the organisation"),
        max_length=1024,
        blank=True,
        null=True,
    )

    description = CKEditor5Field(
        verbose_name=_("Description"),
        help_text=_("Description of the organisation."),
        blank=True,
        null=True,
    )

    principal_place = models.ForeignKey(
        Place,
        verbose_name=_("Principal place"),
        help_text=_("Place mainly associated with this organizer. Mainly used if there is a similarity in the name, to avoid redundant displays."),
        null=True,
        on_delete=models.SET_NULL,
        blank=True,
    )

    class Meta:
        verbose_name = _("Organisation")
        verbose_name_plural = _("Organisations")

    def __str__(self):
        return self.name

    def get_absolute_url(self):
        return reverse("view_organisation", kwargs={'pk': self.pk, "extra": self.name})



class Event(models.Model):
    class STATUS(models.TextChoices):
        PUBLISHED = "published", _("Published")
        DRAFT = "draft", _("Draft")
        TRASH = "trash", _("Trash")

    created_date = models.DateTimeField(editable=False)
    imported_date = models.DateTimeField(blank=True, null=True)
    modified_date = models.DateTimeField(blank=True, null=True)
    moderated_date = models.DateTimeField(blank=True, null=True)

    created_by_user = models.ForeignKey(
        User,
        verbose_name=_("Author of the event creation"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="created_events"
    )
    imported_by_user = models.ForeignKey(
        User,
        verbose_name=_("Author of the last importation"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="imported_events"
    )
    modified_by_user = models.ForeignKey(
        User,
        verbose_name=_("Author of the last modification"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="modified_events"
    )
    moderated_by_user = models.ForeignKey(
        User,
        verbose_name=_("Author of the last moderation"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="moderated_events"
    )

    recurrence_dtstart = models.DateTimeField(editable=False, blank=True, null=True)
    recurrence_dtend = models.DateTimeField(editable=False, blank=True, null=True)

    title = models.CharField(
        verbose_name=_("Title"), max_length=512
    )

    status = models.CharField(
        _("Status"), max_length=20, choices=STATUS.choices, default=STATUS.DRAFT
    )

    category = models.ForeignKey(
        Category,
        verbose_name=_("Category"),
        null=True,
        default=None,
        on_delete=models.SET_DEFAULT,
    )

    start_day = models.DateField(
        verbose_name=_("Start day")
    )
    start_time = models.TimeField(
        verbose_name=_("Start time"),
        blank=True,
        null=True,
    )

    end_day = models.DateField(
        verbose_name=_("End day"),
        blank=True,
        null=True,
    )
    end_time = models.TimeField(
        verbose_name=_("End time"), blank=True, null=True
    )

    recurrences = recurrence.fields.RecurrenceField(
        verbose_name=_("Recurrence"), include_dtstart=False, blank=True, null=True
    )

    exact_location = models.ForeignKey(
        Place,
        verbose_name=_("Location"),
        null=True,
        on_delete=models.SET_NULL,
        blank=True,
    )
    location = models.CharField(
        verbose_name=_("Location (free form)"),
        help_text=_(
            "Address of the event in case its not available in the already known places (free form)"
        ),
        max_length=512,
        default="",
        null=True,
        blank=True
    )

    description = models.TextField(
        verbose_name=_("Description"),
        blank=True,
        null=True,
    )


    organisers = models.ManyToManyField(Organisation, 
        related_name='organised_events',
        verbose_name=_("Organisers"),
        help_text=_(
            "list of event organisers. Organizers will only be displayed if one of them does not normally use the venue."
        ),
        blank=True
    )

    local_image = ResizedImageField(
        size=[900, None],
        verbose_name=_("Illustration"),
        max_length=1024,
        blank=True,
        null=True,
    )

    image = models.URLField(
        verbose_name=_("Illustration (URL)"),
        help_text=_("External URL of the illustration image"),
        max_length=1024,
        blank=True,
        null=True,
    )
    image_alt = models.CharField(
        verbose_name=_("Illustration description"),
        help_text=_("Alternative text used by screen readers for the image"),
        blank=True,
        null=True,
        max_length=1024,
    )

    import_sources = ArrayField(
        models.CharField(max_length=512),
        verbose_name=_("Importation source"),
        help_text=_("Importation source used to detect removed entries."),
        blank=True,
        null=True,
    )
    uuids = ArrayField(
        models.CharField(max_length=512),
        verbose_name=_("UUIDs"),
        help_text=_("UUIDs from import to detect duplicated entries."),
        blank=True,
        null=True,
    )
    reference_urls = ArrayField(
        models.URLField(max_length=512),
        verbose_name=_("Online sources or ticketing"),
        blank=True,
        null=True,
    )

    tags = ArrayField(
        models.CharField(max_length=64),
        verbose_name=_("Tags"),
        blank=True,
        null=True,
    )

    other_versions = models.ForeignKey(
        DuplicatedEvents,
        verbose_name=_("Other versions"),
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
    )

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.processing_user = None
        self._messages = []

    def get_import_messages(self):
        return self.message_set.filter(message_type__in=[Message.TYPE.IMPORT_PROCESS, Message.TYPE.UPDATE_PROCESS]).order_by("date")

    def get_consolidated_end_day(self, intuitive=True):
        if intuitive:
            end_day = self.get_consolidated_end_day(False)
            if end_day != self.start_day and self.end_time and self.end_time < time(8):
                if isinstance(end_day, str):
                    end_day = parse_date(end_day)
                return end_day + timedelta(days=-1)
            else:
                return end_day
        else:
            return self.end_day if self.end_day else self.start_day

    def get_dates(self):
        first = self.start_day
        last = self.get_consolidated_end_day()
        return [first + timedelta(n) for n in range(int((last - first).days) + 1)]

    def is_single_day(self, intuitive=True):
        return self.start_day == self.get_consolidated_end_day(intuitive)

    def is_event_long_duration(self, min_days=4, intuitive=True):
        end_date = self.get_consolidated_end_day(intuitive)
        if isinstance(end_date, str):
            end_date = parse_date(end_date)
        return parse_date(self.start_day) + timedelta(days=min_days) < end_date

    def add_message(self, msg):
        self._messages.append(msg)

    def get_messages(self):
        return self._messages

    def has_message(self):
        return len(self._messages) > 0

    def contains_date(self, d, intuitive=True):
        return d >= self.start_day and d <= self.get_consolidated_end_day(intuitive)

    def get_absolute_url(self):
        return reverse(
            "view_event",
            kwargs={
                "year": self.start_day.year,
                "month": self.start_day.month,
                "day": self.start_day.day,
                "pk": self.pk,
                "extra": slugify(self.title),
            },
        )

    def in_recurrent_import(self):
        if self.import_sources:
            for s in self.import_sources:
                o_s = RecurrentImport.objects.filter(source=s).count()
                if o_s > 0:
                    return True
        return False

    def get_import_sources(self):
        if self.import_sources:
            result = []
            for s in self.import_sources:
                o_s = RecurrentImport.objects.filter(source=s).first()
                if o_s:
                    result.append((o_s.get_absolute_url(), o_s.name))
                else:
                    result.append((s, s))
            return result
        else:
            return []

    def __str__(self):
        return _date(self.start_day) + ": " + self.title

    class Meta:
        verbose_name = _("Event")
        verbose_name_plural = _("Events")
        permissions = [("set_duplicated_event", "Can set an event as duplicated")]
        indexes = [
            models.Index(fields=["start_day", "start_time"]),
            models.Index(fields=["end_day", "end_time"]),
            models.Index(fields=["status"]),
            models.Index(fields=["recurrences"]),
            models.Index(fields=["recurrence_dtstart", "recurrence_dtend"]),
            models.Index("start_time", Lower("title"), name="start_time title"),
            models.Index("start_time", "start_day", "end_day", "end_time", Lower("title"), name="datetimes title")
        ]

    def chronology_dates(self):
        return self.chronology(True)

    def chronology(self, simple=False):
        c = []
        if self.modified_date: 
            c.append({ "timestamp": self.modified_date, "data": "modified_date", "user": self.modified_by_user, "is_date": True })
        if self.moderated_date:
            c.append({ "timestamp": self.moderated_date, "data": "moderated_date", "user" : self.moderated_by_user, "is_date": True})
        if self.imported_date:
            c.append({ "timestamp": self.imported_date, "data": "imported_date", "user": self.imported_by_user, "is_date": True })
        if self.created_date:
            if self.created_by_user:
                user = self.created_by_user
            else:
                if self.in_recurrent_import():
                    user = _('recurrent import')
                else:
                    user = _('a non authenticated user')

            c.append({ "timestamp": self.created_date + timedelta(milliseconds=-1), "data": "created_date", "user": user, "is_date": True})

        if not simple:
            c += [{ "timestamp": m.date, "data": m, "user": m.user, "is_date": False} for m in self.message_set.filter(spam=False)]

            if self.other_versions:
                for o in self.other_versions.get_duplicated():
                    if o != self:
                        c += [{ "timestamp": m.date, "data": m, "user": m.user, "is_date": False} for m in o.message_set.filter(spam=False)]


        c.sort(key=lambda x: x["timestamp"])

        return c

    def sorted_tags(self):
        if self.tags is None:
            return []
        else:
            result = self.tags
            result.sort(key=lambda x: emoji.demojize(remove_accents(x.lower()), delimiters=('000', '')))
            return result

    def get_all_tags(sort=True):
        cursor = connection.cursor()
        raw_query = """
        select unnest(subquery_alias.tags) as distinct_tags, count(*) as tags_group_by_count
            from (select tags from agenda_culturel_event) as subquery_alias group by distinct_tags;
        """
        cursor.execute(raw_query)
        result = [{"tag": row[0], "count": row[1]} for row in cursor]
        if sort:
            result.sort(key=lambda x: emoji.demojize(remove_accents(x["tag"].lower()), delimiters=('000', '')))
        return result

    def is_draft(self):
        return self.status == Event.STATUS.DRAFT

    def is_published(self):
        return self.status == Event.STATUS.PUBLISHED

    def is_trash(self):
        return self.status == Event.STATUS.TRASH

    def modified(self):
        return (
            not self.pure_import()
            and (self.modified_date - self.created_date).total_seconds() > 1
        )
    
    def pure_import(self):
        if self.imported_date is None:
            return False
        return self.modified_date is None or (self.modified_date - self.imported_date).total_seconds() <= 0

    def local_version(self):
        return self.imported_date is None or self.modified()

    def get_reference_urls(self):
        res = [] if self.reference_urls is None else self.reference_urls

        if self.other_versions:
            for o in self.other_versions.get_duplicated():
                if o.status == Event.STATUS.PUBLISHED and not o.reference_urls is None:
                    res += o.reference_urls

        res = list(set(res))
        res.sort()

        return res

    def get_local_version(self):
        # a non-pure import is a local version
        if not self.pure_import():
            return self

        # in case of non other version, return None
        if self.other_versions is None:
            return None

        return self.other_versions.get_local_version()

    def get_shown_organisers(self):
        if self.organisers.count() == 0:
            return None
        if self.exact_location is None:
            has_significant = True
        else:
            has_significant = self.organisers.filter(~Q(principal_place=self.exact_location)).count() > 0

        if has_significant:
            return self.organisers.all()
        else:
            return None

    def get_nb_not_moderated(first_day, nb_mod_days=21, nb_classes=4):
        window_end = first_day + timedelta(days=nb_mod_days)
        nb_not_moderated = Event.objects.filter(~Q(status=Event.STATUS.TRASH)). \
            filter(Q(start_day__gte=first_day)&Q(start_day__lte=window_end)). \
            filter(
                Q(other_versions__isnull=True) |
                Q(other_versions__representative=F('pk')) |
                Q(other_versions__representative__isnull=True)).values("start_day").\
            annotate(not_moderated=Count("start_day", filter=Q(moderated_date__isnull=True))). \
            annotate(nb_events=Count("start_day")). \
            order_by("start_day").values("not_moderated", "nb_events", "start_day")
        
        max_not_moderated = max([x["not_moderated"] for x in nb_not_moderated])
        if max_not_moderated == 0:
            max_not_moderated = 1
        nb_not_moderated_dict = dict([(x["start_day"], (x["not_moderated"], x["nb_events"])) for x in nb_not_moderated])
        # add missing dates
        date_list = [first_day + timedelta(days=x) for x in range(0, nb_mod_days)]
        nb_not_moderated = [{"start_day": d, 
                            "is_today": d == first_day,
                            "nb_events": nb_not_moderated_dict[d][1] if d in nb_not_moderated_dict else 0,
                            "not_moderated": nb_not_moderated_dict[d][0] if d in nb_not_moderated_dict else 0} for d in date_list]
        nb_not_moderated = [ x | { "note": 0 if x["not_moderated"] == 0 else int((nb_classes - 1) * x["not_moderated"] / max_not_moderated) + 1 } for x in nb_not_moderated]
        return [nb_not_moderated[x:x + 7] for x in range(0, len(nb_not_moderated), 7)]


    def nb_draft_events():
        return Event.objects.filter(status=Event.STATUS.DRAFT).count()

    def get_qs_events_with_unkwnon_place():
        return Event.objects.filter(exact_location__isnull=True). \
            filter(~Q(status=Event.STATUS.TRASH)). \
            filter(Q(other_versions=None)|Q(other_versions__representative=F('pk')))

    def is_representative(self):
        return self.other_versions is None or self.other_versions.representative == self

    def download_missing_image(self):
        if self.local_image and not default_storage.exists(self.local_image.name):
            self.download_image()
            self.save(update_fields=["local_image"])

    def download_image(self):
        # first download file

        if str(self.image) != '':
            a = urlparse(self.image)
            basename = os.path.basename(a.path)

            try:
                basename = basename.decode()
            except:
                pass

            ext = basename.split('.')[-1]
            filename = "%s.%s" % (uuid.uuid4(), ext)

            try:
                tmpfile, _ = urllib.request.urlretrieve(self.image)
            except:
                return None

            # if the download is ok, then create the corresponding file object
            self.local_image = File(name=filename, file=open(tmpfile, "rb"))

    def add_pending_organisers(self, organisers):
        self.pending_organisers = organisers
    
    def has_pending_organisers(self):
        return hasattr(self, "pending_organisers")

    def set_is_not_found_import(self):
        self.not_found_import = True

    def is_not_found_import(self):
        return hasattr(self, "not_found_import")

    def set_skip_duplicate_check(self):
        self.skip_duplicate_check = True

    def is_skip_duplicate_check(self):
        return hasattr(self, "skip_duplicate_check")

    def is_in_importation_process(self):
        return hasattr(self, "in_importation_process")

    def set_in_importation_process(self):
        self.in_importation_process = True

    def is_no_modification_date_changed(self):
        return hasattr(self, "no_modification_date_changed")

    def set_no_modification_date_changed(self):
        self.no_modification_date_changed = True

    def set_processing_user(self, user):
        if user is None or user.is_anonymous:
            self.processing_user = None
        else:
            self.processing_user = user

    def set_in_moderation_process(self):
        self.in_moderation_process = True

    def is_in_moderation_process(self):
        return hasattr(self, "in_moderation_process")

    def set_invalid_start_date(self):
        self.invalid_start_date = True

    def has_invalid_start_date(self):
        return hasattr(self, "invalid_start_date")

    def set_invalid_title(self):
        self.invalid_title = True

    def has_invalid_title(self):
        return hasattr(self, "invalid_title")

    def update_modification_dates(self):
        now = timezone.now()
        if not self.id:
            self.created_date = now
            self.created_by_user = self.processing_user
        if self.is_in_importation_process():
            self.imported_date = now
            self.imported_by_user = self.processing_user
        if self.modified_date is None or not self.is_no_modification_date_changed():
            self.modified_date = now
            self.modified_by_user = self.processing_user
        if self.is_in_moderation_process():
            self.moderated_date = now
            self.moderated_by_user = self.processing_user

    def get_recurrence_at_date(self, year, month, day):
        dtstart = timezone.make_aware(
            datetime(year, month, day, 0, 0), timezone.get_default_timezone()
        )
        recurrences = self.get_recurrences_between(dtstart, dtstart)
        if len(recurrences) == 0:
            return self
        else:
            return recurrences[0]

    def get_image_url(self, request=None):
        if self.local_image and hasattr(self.local_image, "url"):
            try:
                if request:
                    return request.build_absolute_uri(self.local_image.url)
                else:
                    return self.local_image.url
            except:
                pass
        if self.image:
            return self.image
        else:
            return None

    def has_image_url(self):
        return self.get_image_url() is not None


    # return a copy of the current object for each recurrence between first an last date (included)
    def get_recurrences_between(self, firstdate, lastdate):
        if not self.has_recurrences():
            return [self]
        else:
            result = []
            dtstart = timezone.make_aware(
                datetime.combine(self.start_day, time()),
                timezone.get_default_timezone(),
            )
            self.recurrences.dtstart = dtstart
            for d in self.recurrences.between(
                firstdate, lastdate, inc=True, dtstart=dtstart
            ):
                c = copy.deepcopy(self)
                c.start_day = d.date()
                if c.end_day is not None:
                    shift = d.date() - self.start_day
                    c.end_day += shift
                result.append(c)

            return result

    def has_recurrences(self):
        # TODO: see https://forge.chapril.org/jmtrivial/agenda_culturel/issues/65
        return self.recurrences is not None and len(self.recurrences.rrules) != 0

    def update_recurrence_dtstartend(self):
        sday = (
            date.fromisoformat(self.start_day)
            if isinstance(self.start_day, str)
            else self.start_day
        )
        eday = (
            date.fromisoformat(self.end_day)
            if isinstance(self.end_day, str)
            else self.end_day
        )
        stime = (
            time.fromisoformat(self.start_time)
            if isinstance(self.start_time, str)
            else time()
            if self.start_time is None
            else self.start_time
        )
        etime = (
            time.fromisoformat(self.end_time)
            if isinstance(self.end_time, str)
            else time()
            if self.end_time is None
            else self.end_time
        )

        self.recurrence_dtstart = datetime.combine(sday, stime)
        if not self.has_recurrences():
            if self.end_day is None:
                self.dtend = None
            else:
                self.recurrence_dtend = datetime.combine(eday, etime)
        else:
            if (
                self.recurrences.rrules[0].until is None
                and self.recurrences.rrules[0].count is None
            ):
                self.recurrence_dtend = None
            else:
                self.recurrences.dtstart = datetime.combine(sday, time())
                occurrence = self.recurrences.occurrences()
                try:
                    self.recurrence_dtend = occurrence[-1]
                    if (
                        self.recurrences.dtend is not None
                        and self.recurrences.dtstart is not None
                    ):
                        self.recurrence_dtend += (
                            self.recurrences.dtend - self.recurrences.dtstart
                        )
                except:
                    self.recurrence_dtend = self.recurrence_dtstart

    def prepare_save(self):
        self.update_modification_dates()

        self.update_recurrence_dtstartend()

        # if the image is defined but not locally downloaded
        if self.image and (not self.local_image or not default_storage.exists(self.local_image.name)):
            self.download_image()

        # remove "/" from tags
        if self.tags:
            self.tags = [t.replace('/', '-') for t in self.tags]

        # in case of importation process
        if self.is_in_importation_process():
            # try to detect location
            if not self.exact_location:
                for p in Place.objects.all():
                    if p.match(self):
                        self.exact_location = p
                        break
            # try to detect category
            if not self.category or self.category.name == Category.default_name:
                CategorisationRule.apply_rules(self)


    def get_contributor_message(self):
        types = [Message.TYPE.FROM_CONTRIBUTOR, Message.TYPE.FROM_CONTRIBUTOR_NO_MSG]
        if self.other_versions is None or self.other_versions.representative is None:
            return Message.objects.filter(related_event=self.pk, message_type__in=types, closed=False)
        else:
            return Message.objects.filter(related_event__in=self.other_versions.get_duplicated(), message_type__in=types, closed=False)


    def notify_if_required(self, request):
        notif = False
        if self.status != Event.STATUS.DRAFT:
            messages = self.get_contributor_message()
            if messages:
                for message in messages:
                    if message and not message.closed and message.email and message.email != "":
                        # send email
                        context = {"sitename": Site.objects.get_current(request).name, 'event_title': self.title }
                        if self.status == Event.STATUS.PUBLISHED:
                            context["url"] = request.build_absolute_uri(self.get_absolute_url())
                            subject = _('Your event has been published')
                            body = render_to_string("agenda_culturel/emails/published.txt", context)
                        else:
                            subject = _('Your message has not been retained')
                            body = render_to_string("agenda_culturel/emails/retained.txt", context)

                        send_mail(subject, body, None, [message.email])
                        message.closed = True
                        message.save()
                        notif = True

        return notif


    def save(self, *args, **kwargs):
        self.prepare_save()

        # check for similar events if no duplicated is known and only if the event is created
        if (
            self.pk is None
            and self.other_versions is None
            and not self.is_skip_duplicate_check()
        ):
            # and if this is not an importation process
            if not self.is_in_importation_process():
                similar_events = self.find_similar_events()

                # if it exists similar events, add this relation to the event
                if len(similar_events) != 0:
                    self.set_other_versions(similar_events)
        
        # check if it's a clone (that will become representative)
        clone = self.pk is None and not self.other_versions is None

        # check if we need to clean the other_versions
        if (
            not clone and
            self.pk and
            self.other_versions is not None
            and self.other_versions.nb_duplicated() == 1
        ):
            self.other_versions.delete()
            self.other_versions = None

        # first save the current object
        super().save(*args, **kwargs)

        # notify only if required (and request is known)
        if "request" in kwargs:
            self.notify_if_required(kwargs.get("request"))

        # clear cache
        for is_auth in [False, True]:
            key = make_template_fragment_key("event_body", [is_auth, self])
            cache.delete(key)

        # save message if required
        if self.has_message():
            for msg in self.get_messages():
                msg.related_event = self
                msg.save()

        # then if its a clone, update the representative
        if clone:
            self.other_versions.representative = self
            self.other_versions.save()
            # if we just clone a single event, its status is 
            # the same as the status of the current object
            if self.other_versions.get_duplicated().count() == 2:
                for e in self.other_versions.get_duplicated():
                    if e != self:
                        e.status = self.status
                        e.set_no_modification_date_changed()
                        e.save()

    def from_structure(event_structure, import_source=None):
        # organisers is a manytomany relation thus cannot be initialised before creation of the event
        organisers = event_structure.pop('organisers', None)
        # supplementary information
        email = event_structure.pop('email', None)
        comments = event_structure.pop('comments', None)
        warnings = event_structure.pop('warnings', [])

        for w in warnings:
            if w == Extractor.Warning.NO_START_DATE:
                event_structure["title"] += " - " + _('Warning') + ": " + _('the date has not been imported correctly.')


        if "category" in event_structure and event_structure["category"] is not None:
            try:
                event_structure["category"] = Category.objects.get(
                    name__unaccent__icontains=remove_accents(event_structure["category"].lower())
                )
            except Category.DoesNotExist:
                event_structure["category"] = Category.get_default_category()
        else:
            event_structure["category"] = Category.get_default_category()

        if "published" in event_structure and event_structure["published"] is not None:
            if event_structure["published"]:
                event_structure["status"] = Event.STATUS.PUBLISHED
            else:
                event_structure["status"] = Event.STATUS.DRAFT
            del event_structure["published"]
        else:
            event_structure["status"] = Event.STATUS.DRAFT

        if "url_human" in event_structure and event_structure["url_human"] is not None:
            event_structure["reference_urls"] = [event_structure["url_human"]]
            del event_structure["url_human"]

        if (
            "last_modified" in event_structure
            and event_structure["last_modified"] is not None
        ):
            d = datetime.fromisoformat(event_structure["last_modified"])
            if d.tzinfo is None or d.tzinfo.utcoffset(d) is None:
                d = timezone.make_aware(d, timezone.get_default_timezone())
            event_structure["modified_date"] = d
            del event_structure["last_modified"]
        else:
            event_structure["modified_date"] = None

        if "start_time" in event_structure:
            event_structure["start_time"] = time.fromisoformat(
                event_structure["start_time"]
            )

        if "end_time" in event_structure:
            event_structure["end_time"] = time.fromisoformat(
                event_structure["end_time"]
            )

        if "location" not in event_structure or event_structure["location"] is None:
            event_structure["location"] = ""

        if "description" in event_structure and event_structure["description"] is None:
            event_structure["description"] = ""

        if (
            "recurrences" in event_structure
            and event_structure["recurrences"] is not None
        ):
            event_structure["recurrences"] = recurrence.deserialize(
                event_structure["recurrences"]
            )
            event_structure["recurrences"].exdates = [
                e.replace(hour=0, minute=0, second=0)
                for e in event_structure["recurrences"].exdates
            ]
            event_structure["recurrences"].rdates = [
                e.replace(hour=0, minute=0, second=0)
                for e in event_structure["recurrences"].rdates
            ]

        else:
            event_structure["recurrences"] = None

        if import_source is not None:
            event_structure["import_sources"] = [import_source]

        result = Event(**event_structure)
        result.add_pending_organisers(organisers)
        if email or comments:
            has_comments = not comments in ["", None]
            result.add_message(Message(subject=_('during import process'),
                               email=email,
                               message=comments,
                               closed=False,
                               message_type=Message.TYPE.FROM_CONTRIBUTOR if has_comments else Message.TYPE.FROM_CONTRIBUTOR_NO_MSG))
        for w in warnings:
            if w == Extractor.Warning.NO_START_DATE:
                result.set_invalid_start_date()
                result.add_message(Message(subject=_('warning'),
                                closed=False,
                                message=_('the date has not been imported correctly.'),
                                message_type=Message.TYPE.WARNING))
            if w == Extractor.Warning.NO_TITLE:
                result.set_invalid_title()
                result.add_message(Message(subject=_('warning'),
                                closed=False,
                                message=_('the title has not been imported correctly.'),
                                message_type=Message.TYPE.WARNING))
            if w == Extractor.Warning.NOT_FOUND:
                result.status = Event.STATUS.DRAFT
                result.set_is_not_found_import()
                result.add_message(Message(subject=_('warning'),
                                closed=False,
                                message=_('The import was unable to find an event in the page.'),
                                message_type=Message.TYPE.WARNING))

        return result


    def find_similar_events(self):
        start_time_test = Q(start_time=self.start_time)

        if self.start_time is not None:
            # convert str start_time to time
            if isinstance(self.start_time, str):
                self.start_time = time.fromisoformat(self.start_time)
            interval = (
                time(self.start_time.hour - 1, self.start_time.minute)
                if self.start_time.hour >= 1
                else time(0, 0),
                time(self.start_time.hour + 1, self.start_time.minute)
                if self.start_time.hour < 23
                else time(23, 59),
            )
            start_time_test = start_time_test | Q(start_time__range=interval)

        return (
            Event.objects.annotate(
                similarity_title=TrigramSimilarity("title", self.title)
            )
            .annotate(similarity_location=TrigramSimilarity("location", self.location))
            .filter(
                Q(start_day=self.start_day)
                & start_time_test
                & Q(similarity_title__gt=0.5)
                & Q(similarity_title__gt=0.3)
            )
        )

    def find_same_events_by_uuid(self):
        return (
            None
            if self.uuids is None or len(self.uuids) == 0
            else Event.objects.filter(uuids__contains=self.uuids)
        )
    
    def get_updateable_uuid(self):
        if self.uuids and len(self.uuids) > 0:
            for s in self.uuids:
                if FacebookEventExtractor.is_known_url(s):
                    return s
        return None

    def is_updateable(self):
        return not self.get_updateable_uuid() is None

    def split_uuid(uuid):
        els = uuid.split(":")
        if len(els) == 1:
            return ":".join(els[0:-1]), 0
        else:
            if els[-1].isdigit():
                return ":".join(els[0:-1]), int(els[-1])
            else:
                return ":".join(els), 0

    def is_ancestor_uuid(uuid1, uuid2):
        root1, version1 = Event.split_uuid(uuid1)
        root2, version2 = Event.split_uuid(uuid2)
        return root1 == root2 and version1 < version2

    def is_ancestor_by_uuid(self, event):
        if self.uuids is None or event.uuids is None:
            return False

        for s_uuid in self.uuids:
            for e_uuid in event.uuids:
                if Event.is_ancestor_uuid(s_uuid, e_uuid):
                    return True
        return False
    
    def same_uuid(self, event):
        if self.uuids is None or event.uuids is None:
            return False
        for s_uuid in self.uuids:
            for e_uuid in event.uuids:
                if s_uuid == e_uuid:
                    return True
        return False

    def get_other_not_trash_versions(self):
        if self.other_versions is None:
            return []
        else:
            return [e for e in self.other_versions.get_duplicated() if e.pk != self.pk and e.status != Event.STATUS.TRASH]

    def get_other_versions(self):
        if self.other_versions is None:
            return []
        else:
            return [e for e in self.other_versions.get_duplicated() if e.pk != self.pk]

    def masked(self):
        return self.other_versions and self.other_versions.representative != None and self.other_versions.representative.pk != self.pk

    def get_organisers(self):
        if self.pk:
            if self.organisers is None:
                return []
            else:
                return self.organisers.all()
        else:
            if self.has_pending_organisers():
                if self.pending_organisers is None:
                    return []
                else:
                    return self.pending_organisers
            else:
                return []
            

    def get_comparison(events, all=True):
        result = []
        for attr in Event.data_fields(all=all, local_img=False, exact_location=False):
            if attr == 'organisers':
                values = [[str(o) for o in e.get_organisers()] for e in events]
            else:
                values = [getattr(e, attr) for e in events]
            values = ["" if v is None else v for v in values]
            values = [[] if attr == "tags" and v == "" else v for v in values]
            # only consider fixed part of Facebook urls
            if attr == "image":
                values = [v.split("?")[0] if "fbcdn.net" in v else v for v in values]

            if attr == "description":
                values = [v.replace("\r\n", "\n") for v in values]
            if len(set([str(v).strip() for v in values])) == 1:
                result.append({"similar": True, "key": attr, "values": values[0]})
            else:
                result.append({"similar": False, "key": attr, "values": values})
        return result

    def similar(self, event, all=True):
        res = Event.get_comparison([self, event], all)
        for r in res:
            if not r["similar"]:
                if r["key"] == "title" and (self.has_invalid_title() or event.has_invalid_title()):
                    continue
                if r["key"] == "start_day" and (self.has_invalid_start_date() or event.has_invalid_start_date()):
                    continue
                return False
        return True

    def set_other_versions(self, events, force_non_fixed=False):
        # get existing groups
        groups = list(
            set([e.other_versions for e in events] + [self.other_versions])
        )
        groups = [g for g in groups if g is not None]

        # do we have to create a new group?
        if len(groups) == 0:
            group = DuplicatedEvents.objects.create()
        else:
            # otherwise merge existing groups
            group = DuplicatedEvents.merge_groups(groups)

        group.save()

        if force_non_fixed:
            group.representative = None

        # set the possibly duplicated group for the current object
        self.other_versions = group

        # and for the other events
        for e in events:
            e.other_versions = group

        # finally update all events (including current if already created)
        elist = list(events) + ([self] if self.pk is not None else [])
        Event.objects.bulk_update(elist, fields=["other_versions"])

    def data_fields(local_img=True, exact_location=True, all=True, no_m2m=False):
        result = []

        if all:
            result += [
                "category",
                "tags",
            ]
            if not no_m2m:
                result += ["organisers"]

        result += [
            "title",
            "location",
            "start_day",
            "start_time",
            "end_day",
            "end_time",
            "description",
            "image",
        ]
        if all and local_img:
            result += ["local_image"]
        if all and exact_location:
            result += ["exact_location"]
        result += ["image_alt", "reference_urls", "recurrences"]

        return result

    def find_last_imported(events):
        events = [e for e in events if e.imported_date is not None]
        if len(events) == 0:
            return None
        else:
            events.sort(key=lambda e: e.imported_date, reverse=True)
            return events[0]

    def find_last_pure_import(events):
        events = [e for e in events if e.pure_import()]
        if len(events) == 0:
            return None
        else:
            events.sort(key=lambda e: e.imported_date, reverse=True)
            return events[0]

    def update(self, other, all):

        # integrate pending organisers
        if other.has_pending_organisers() and not other.pending_organisers is None:
            self.organisers.set(other.pending_organisers)
        
        logger.warning("process update " + other.title + ' ' + str(other.has_invalid_start_date()))
        # set attributes
        for attr in Event.data_fields(all=all, no_m2m=True):
            if attr == "title" and other.has_invalid_title():
                continue
            if attr == "start_day" and other.has_invalid_start_date():
                continue
            setattr(self, attr, getattr(other, attr))

        # adjust modified date if required
        if other.modified_date and self.modified_date < other.modified_date:
            self.modified_date = other.modified_date

        # add a possible missing uuid
        if self.uuids is None:
            self.uuids = []
        for uuid in other.uuids:
            if uuid not in self.uuids:
                self.uuids.append(uuid)

        # add possible missing sources
        if other.import_sources:
            if not self.import_sources:
                self.import_sources = []
            for source in other.import_sources:
                if source not in self.import_sources:
                    self.import_sources.append(source)

    # Limitation: the given events should not be considered similar one to another...
    def import_events(events, remove_missing_from_source=None, user_id=None):
        user = None
        if user_id:
            user = User.objects.filter(pk=user_id).first()

        to_import = []
        to_update = []

        min_date = timezone.now().date()
        max_date = None
        uuids = set()

        # for each event, check if it's a new one, or a one to be updated
        for event in events:
            sdate = date.fromisoformat(event.start_day)
            if event.end_day:
                edate = date.fromisoformat(event.end_day)
            else:
                edate = sdate
            if min_date is None or min_date > sdate:
                min_date = sdate
            if max_date is None or max_date < sdate:
                max_date = sdate
            if max_date is None or (event.end_day is not None and max_date < edate):
                max_date = edate
            if event.uuids and len(event.uuids) > 0:
                uuids |= set(event.uuids)

            # imported events should be updated
            event.set_in_importation_process()
            event.set_processing_user(user)
            event.prepare_save()

            # check if the event has already be imported (using uuid)
            same_events = event.find_same_events_by_uuid()

            if len(same_events) != 0:
                # check if one event has been imported and not modified in this list
                same_imported = Event.find_last_pure_import(same_events)
                pure = True
                # if not, we check if it does not match exactly with another
                if not same_imported:
                    for e in same_events:
                        if event.similar(e, False):
                            same_imported = e
                            pure = False
                            break

                if same_imported:
                    if not event.similar(same_imported, False):
                        # reopen DuplicatedEvents if required
                        if same_imported.other_versions:
                            if same_imported.status != Event.STATUS.TRASH:
                                if same_imported.other_versions.is_published():
                                    if same_imported.other_versions.representative != same_imported:
                                        same_imported.other_versions.representative = None
                                        same_imported.other_versions.save()
                        # add a message to explain the update
                        if not event.is_not_found_import():
                            res = [r for r in Event.get_comparison([event, same_imported], all) if not r["similar"]]
                            if len(res) > 0:
                                txt = _("Updated field(s): ") + ", ".join([r["key"] for r in res])
                                msg = Message(subject=_('Update'), 
                                        name=_('update process'),
                                        related_event=same_imported,
                                        message=txt,
                                        message_type=Message.TYPE.UPDATE_PROCESS)
                                msg.save()

                    new_image = same_imported.image != event.image

                    # if the import process was not able to found any content, change the status as draft
                    if event.is_not_found_import():
                        if same_imported.status == Event.STATUS.PUBLISHED:
                            same_imported.status = Event.STATUS.TRASH
                    else:
                        # we only update local information if it's a pure import and has no moderated_date
                        same_imported.update(event, pure and same_imported.moderated_date is None) 
                    # save messages
                    if event.has_message():
                        for msg in event.get_messages():
                            msg.related_event = same_imported
                            msg.save()
                    same_imported.set_in_importation_process()
                    same_imported.prepare_save()
                    # fix missing or updated files
                    if same_imported.local_image and (not default_storage.exists(same_imported.local_image.name) or new_image):
                        same_imported.download_image()
                        same_imported.save(update_fields=["local_image"])
                    to_update.append(same_imported)
                else:
                    # otherwise, the new event possibly a duplication of the remaining others.

                    # check if it should be published
                    trash = len([e for e in same_events if e.status != Event.STATUS.TRASH]) == 0
                    if trash:
                        event.status = Event.STATUS.TRASH
                    event.set_other_versions(same_events, force_non_fixed=not trash)
                    # it will be imported
                    to_import.append(event)
            else:
                # if uuid is unique (or not available), check for similar events
                similar_events = event.find_similar_events()

                # if it exists similar events, add this relation to the event
                if len(similar_events) != 0:
                    # the event is possibly a duplication of the others
                    event.set_other_versions(similar_events, force_non_fixed=True)
                    to_import.append(event)
                else:
                    # import this new event
                    to_import.append(event)

        for e in to_import:
            if e.is_event_long_duration():
                e.status = Event.STATUS.DRAFT
                e.add_message(
                    Message(subject=_('Import'), 
                            name=_('import process'),
                            message=_("The duration of the event is a little too long for direct publication. Moderators can choose to publish it or not."),
                            message_type=Message.TYPE.IMPORT_PROCESS)
                )

        # then import all the new events
        imported = Event.objects.bulk_create(to_import)
        # update organisers (m2m relation)
        for i, ti in zip(imported, to_import):
            if ti.has_pending_organisers() and ti.pending_organisers is not None:
                i.organisers.set(ti.pending_organisers)
            if ti.has_message():
                for msg in ti.get_messages():
                    msg.related_event = i
                    msg.save()

        nb_updated = Event.objects.bulk_update(
            to_update,
            fields=Event.data_fields(no_m2m=True)
            + ["imported_date", "modified_date", "uuids", "status", "import_sources"],
        )

        nb_draft = 0
        if remove_missing_from_source is not None and max_date is not None:
            # events that are missing from the import but in database are turned into drafts
            # only if they are in the future

            in_interval = Event.objects.filter(
                (
                    (
                        Q(end_day__isnull=True)
                        & Q(start_day__gte=min_date)
                        & Q(start_day__lte=max_date)
                    )
                    | (
                        Q(end_day__isnull=False)
                        & ~(Q(start_day__gt=max_date) | Q(end_day__lt=min_date))
                    )
                )
                & Q(import_sources__contains=[remove_missing_from_source])
                & Q(status=Event.STATUS.PUBLISHED)
                & Q(uuids__len__gt=0)
            )

            to_draft = []
            for e in in_interval:
                if len(uuids.intersection(e.uuids)) == 0:
                    e.status = Event.STATUS.TRASH
                    # save them without updating modified date
                    e.set_no_modification_date_changed()
                    e.prepare_save()
                    to_draft.append(e)

            nb_draft = Event.objects.bulk_update(to_draft, fields=["status"])

        return imported, nb_updated, nb_draft

    def set_current_date(self, date):
        self.current_date = date

    def get_start_end_datetimes(self, day):
        start_h = time().min
        if self.start_day == day:
            if self.start_time is None:
                dtstart = datetime.combine(self.start_day, time().min)
            else:
                dtstart = datetime.combine(self.start_day, self.start_time)
                start_h = self.start_time
        else:
            dtstart = datetime.combine(day, time().min)

        end_day = self.get_consolidated_end_day()

        if end_day == day:
            if self.end_time is None:
                if end_day == self.start_day:
                    dtend = dtstart + timedelta(hours=2)
                else:
                    dtend = datetime.combine(end_day, time().max)
            else:
                dtend = datetime.combine(end_day, self.end_time)
        else:
            dtend = datetime.combine(day, time().max)

        return dtstart, dtend

    def get_concurrent_events(self, remove_same_dup=True):
        day = self.current_date if hasattr(self, "current_date") else self.start_day
        day_events = CalendarDay(day, qs = Event.objects.filter(status=Event.STATUS.PUBLISHED).only("start_day", "start_time", "title", "category", "other_versions", "recurrences", "end_day", "end_time", "uuids")).get_events()
        return [
            e
            for e in day_events
            if e != self
            and self.is_concurrent_event(e, day)
            and (e.other_versions is None or e.other_versions != self.other_versions)
        ]

    def is_concurrent_event(self, e, day):
        dtstart, dtend = self.get_start_end_datetimes(day)
        e_dtstart, e_dtend = e.get_start_end_datetimes(day)

        return (dtstart <= e_dtstart <= dtend) or (e_dtstart <= dtstart <= e_dtend)

    def export_to_ics(events, request):
        cal = icalCal()
        # Some properties are required to be compliant
        cal.add("prodid", "-//Pommes de lune//pommesdelune.fr//")
        cal.add("version", "2.0")

        for event in events:
            ed = event.get_consolidated_end_day()
            eventIcal = icalEvent()
            # mapping
            if event.start_time is None:
                eventIcal.add(
                    "dtstart",
                    date(
                        event.start_day.year,
                        event.start_day.month,
                        event.start_day.day,
                    ),
                )
            else:
                eventIcal.add(
                    "dtstart",
                    datetime(
                        event.start_day.year,
                        event.start_day.month,
                        event.start_day.day,
                        event.start_time.hour,
                        event.start_time.minute,
                    ),
                )
            if not event.end_day is None:
                if event.end_time is None:
                    eventIcal.add(
                        "dtend",
                        date(
                            event.end_day.year,
                            event.end_day.month,
                            event.end_day.day,
                        ),
                    )
                else:
                    eventIcal.add(
                        "dtend",
                        datetime(
                            event.end_day.year,
                            event.end_day.month,
                            event.end_day.day,
                            event.end_time.hour,
                            event.end_time.minute,
                        ),
                    )
            eventIcal.add("summary", event.title)
            eventIcal.add("name", event.title)
            url = ("\n" + event.reference_urls[0]) if event.reference_urls and len(event.reference_urls) > 0 else ""
            description = event.description if event.description else ""
            eventIcal.add(
                "description", description + url
            )
            if not event.local_image is None and event.local_image != "":
                eventIcal.add('image', request.build_absolute_uri(event.local_image), parameters={'VALUE': 'URI'})
            eventIcal.add("location", event.exact_location or event.location)

            cal.add_component(eventIcal)

        return cal

    def get_count_modification(when):
        start = datetime(when[0].year, when[0].month, when[0].day)
        end   = start + timedelta(days=when[1])

        recurrentimport = RecurrentImport.objects.filter(source=OuterRef("import_sources__0")).order_by().annotate(count=Func(F('id'), function='Count')).values('count')


        nb_manual_creation = Event.objects.filter(created_date__gte=start, created_date__lt=end, imported_date__isnull=True).filter(Q(uuids__len=0)|Q(uuids=None)).count()
        nb_local_copies = Event.objects.filter(created_date__gte=start, created_date__lt=end, imported_date__isnull=True, uuids__len__gt=0).count()

        imported = Event.objects.filter(created_date__gte=start, created_date__lt=end, imported_date__isnull=False, uuids__len__gt=0).annotate(nb_rimport=Subquery(recurrentimport))
        
        nb_manual_import = imported.filter(Q(import_sources__len=0)|Q(nb_rimport=0)).count()
        nb_first_import = imported.filter(nb_rimport__gt=0).count()

        nb_moderated = Event.objects.filter(moderated_date__gte=start, moderated_date__lt=end).count()

        return {"when": (start, start + timedelta(days=when[1] - 1)), 
                "nb_manual_creation": nb_manual_creation, 
                "nb_manual_import": nb_manual_import, 
                "nb_first_import": nb_first_import, 
                "nb_local_copies": nb_local_copies,
                "nb_moderated": nb_moderated,
        }

    def get_count_modifications(when_list):
        return [Event.get_count_modification(w) for w in when_list]


class Message(models.Model):

    class TYPE(models.TextChoices):
        FROM_CONTRIBUTOR = "from_contributor", _("From contributor")
        IMPORT_PROCESS = "import_process", _("Import process")
        UPDATE_PROCESS = "update_process", _("Update process")
        CONTACT_FORM = "contact_form", _("Contact form")
        EVENT_REPORT = "event_report", _("Event report")
        FROM_CONTRIBUTOR_NO_MSG = "from_contrib_no_msg", _("From contributor (without message)")
        WARNING = "warning", _("Warning")

    class Meta:
        verbose_name = _("Message")
        verbose_name_plural = _("Messages")
        indexes = [
            models.Index(fields=['related_event']),
            models.Index(fields=['user']),
            models.Index(fields=['date']),
            models.Index(fields=['spam', 'closed']),
        ]



    subject = models.CharField(
        verbose_name=_("Subject"),
        help_text=_("The subject of your message"),
        max_length=512,
    )

    related_event = models.ForeignKey(
        Event,
        verbose_name=_("Related event"),
        help_text=_("The message is associated with this event."),
        null=True,
        default=None,
        on_delete=models.SET_DEFAULT,
    )

    user = models.ForeignKey(
        User,
        verbose_name=_("Author of the message"),
        null=True,
        default=None,
        on_delete=models.SET_DEFAULT,
    )

    name = models.CharField(
        verbose_name=_("Name"),
        help_text=_("Your name"),
        max_length=512,
        blank=True,
        null=True,
    )
    email = models.EmailField(
        verbose_name=_("Email address"),
        help_text=_("Your email address"),
        max_length=254,
        blank=True,
        null=True,
    )
    message = CKEditor5Field(verbose_name=_("Message"), help_text=_("Your message"), blank=True)

    date = models.DateTimeField(auto_now_add=True)

    spam = models.BooleanField(
        verbose_name=_("Spam"),
        help_text=_("This message is a spam."),
        default=False,
    )

    closed = models.BooleanField(
        verbose_name=_("Closed"),
        help_text=_(
            "this message has been processed and no longer needs to be handled"
        ),
        default=False,
    )
    comments = CKEditor5Field(
        verbose_name=_("Comments"),
        help_text=_("Comments on the message from the moderation team"),
        default="",
        blank=True,
        null=True,
    )

    message_type = models.CharField(
        verbose_name=_("Type"),
        max_length=20,
        choices=TYPE.choices,
        default=None, null=True
    )

    def nb_open_messages():
        return Message.objects.filter(Q(closed=False)&Q(spam=False)&Q(message_type__in=[Message.TYPE.CONTACT_FORM, Message.TYPE.EVENT_REPORT, Message.TYPE.FROM_CONTRIBUTOR])).count()

    def get_absolute_url(self):
        return reverse("message", kwargs={"pk": self.pk})


class RecurrentImport(models.Model):
    class Meta:
        verbose_name = _("Recurrent import")
        verbose_name_plural = _("Recurrent imports")
        permissions = [("run_recurrentimport", "Can run a recurrent import")]

    class PROCESSOR(models.TextChoices):
        ICAL = "ical", _("ical")
        ICALNOBUSY = "icalnobusy", _("ical no busy")
        ICALNOVC = "icalnovc", _("ical no VC")
        ICALNAIVETZ = "ical naive tz", _("ical naive timezone")
        LACOOPE = "lacoope", _("lacoope.org")
        LACOMEDIE = "lacomedie", _("la comédie")
        LEFOTOMAT = "lefotomat", _("le fotomat")
        LAPUCEALOREILLE = "lapucealoreille", _("la puce à l'oreille")
        MECWORDPRESS = "Plugin wordpress MEC", _("Plugin wordpress MEC")
        FBEVENTS = "Facebook events", _("Événements d'une page FB")
        BILLETTERIECF = "Billetterie CF", _("Billetterie Clermont-Ferrand")
        ARACHNEE = "arachnee", _("Arachnée concert")
        LERIO = "rio", _('Le Rio')
        LARAYMONDE = "raymonde", _('La Raymonde')
        APIDAE = 'apidae', _('Agenda apidae tourisme')
        IGUANA = 'iguana', _('Agenda iguana (médiathèques)')
        MILLEFORMES = 'Mille formes', _('Mille formes')
        AMISCERISES = 'Amis cerises', _('Les Amis du Temps des Cerises')
        MOBILIZON = 'Mobilizon', _('Mobilizon')

    class DOWNLOADER(models.TextChoices):
        SIMPLE = "simple", _("simple")
        CHROMIUMHEADLESS = "chromium headless", _("Headless Chromium")
        CHROMIUMHEADLESSPAUSE = "chromium (pause)", _("Headless Chromium (pause)")

    class RECURRENCE(models.TextChoices):
        DAILY = (
            "daily",
            _("daily"),
        )
        WEEKLY = "weekly", _("weekly")

    name = models.CharField(
        verbose_name=_("Name"),
        help_text=_(
            "Recurrent import name. Be careful to choose a name that is easy to understand, as it will be public and displayed on the site"
            "s About page."
        ),
        max_length=512,
        default="",
    )
    processor = models.CharField(
        _("Processor"), max_length=20, choices=PROCESSOR.choices, default=PROCESSOR.ICAL
    )
    downloader = models.CharField(
        _("Downloader"),
        max_length=20,
        choices=DOWNLOADER.choices,
        default=DOWNLOADER.SIMPLE,
    )

    recurrence = models.CharField(
        _("Import recurrence"),
        max_length=10,
        choices=RECURRENCE.choices,
        default=RECURRENCE.DAILY,
    )

    source = models.URLField(
        verbose_name=_("Source"),
        help_text=_("URL of the source document"),
        unique=True,
        max_length=1024,
    )
    browsable_url = models.URLField(
        verbose_name=_("Browsable url"),
        help_text=_(
            "URL of the corresponding document that will be shown to visitors."
        ),
        max_length=1024,
        blank=True,
        null=True,
    )

    defaultPublished = models.BooleanField(
        verbose_name=_("Published"),
        help_text=_("Status of each imported event (published or draft)"),
        default=True,
    )
    defaultLocation = models.CharField(
        verbose_name=_("Location"),
        help_text=_("Address for each imported event"),
        max_length=512,
        null=True,
        blank=True,
    )

    forceLocation = models.BooleanField(
        verbose_name=_("Force location"),
        help_text=_("force location even if another is detected."),
        default=False
    )

    defaultOrganiser = models.ForeignKey(
        Organisation,
        verbose_name=_("Organiser"),
        help_text=_("Organiser of each imported event"),
        default=None,
        null=True,
        blank=True,
        on_delete=models.SET_DEFAULT,
    )

    defaultCategory = models.ForeignKey(
        Category,
        verbose_name=_("Category"),
        help_text=_("Category of each imported event"),
        default=None,
        null=True,
        blank=True,
        on_delete=models.SET_DEFAULT,
    )
    defaultTags = ArrayField(
        models.CharField(max_length=64),
        verbose_name=_("Tags for each imported event"),
        help_text=_("A list of tags that describe each imported event."),
        blank=True,
        null=True,
    )

    def __str__(self):
        return self.name

    def nb_imports(self):
        return BatchImportation.objects.filter(recurrentImport=self).count()

    def nb_events(self):
        return Event.objects.filter(import_sources__contains=[self.source]).count()

    def get_absolute_url(self):
        return reverse("view_rimport", kwargs={"pk": self.pk})

    def last_import(self):
        events = BatchImportation.objects.filter(recurrentImport=self).order_by(
            "-created_date"
        )
        if events and len(events) > 0:
            return events[0]
        else:
            return None


class BatchImportation(models.Model):
    class STATUS(models.TextChoices):
        RUNNING = "running", _("Running")
        CANCELED = "canceled", _("Canceled")
        SUCCESS = "success", _("Success")
        FAILED = "failed", _("Failed")

    class Meta:
        verbose_name = _("Batch importation")
        verbose_name_plural = _("Batch importations")
        permissions = [("run_batchimportation", "Can run a batch importation")]
        indexes = [
            models.Index(fields=['created_date']),
            models.Index(fields=['status']),
            models.Index(fields=['created_date', 'recurrentImport']),
        ]

    created_date = models.DateTimeField(auto_now_add=True)

    recurrentImport = models.ForeignKey(
        RecurrentImport,
        verbose_name=_("Recurrent import"),
        help_text=_("Reference to the recurrent import processing"),
        blank=True,
        null=True,
        on_delete=models.SET_NULL,
        editable=False,
    )

    url_source = models.URLField(
        verbose_name=_("URL (if not recurrent import)"),
        help_text=_(
            "Source URL if no RecurrentImport is associated."
        ),
        max_length=1024,
        blank=True,
        null=True,
        editable=False,
    )

    status = models.CharField(
        _("Status"), max_length=20, choices=STATUS.choices, default=STATUS.RUNNING
    )

    error_message = models.CharField(
        verbose_name=_("Error message"), max_length=512, blank=True, null=True
    )

    nb_initial = models.PositiveIntegerField(
        verbose_name=_("Number of collected events"), default=0
    )
    nb_imported = models.PositiveIntegerField(
        verbose_name=_("Number of imported events"), default=0
    )
    nb_updated = models.PositiveIntegerField(
        verbose_name=_("Number of updated events"), default=0
    )
    nb_removed = models.PositiveIntegerField(
        verbose_name=_("Number of removed events"), default=0
    )

    celery_id = models.CharField(max_length=128, default="")


class CategorisationRule(models.Model):
    weight = models.IntegerField(
        verbose_name=_("Weight"),
        help_text=_("The lower is the weight, the earlier the filter is applied"),
        default=1,
    )

    category = models.ForeignKey(
        Category,
        verbose_name=_("Category"),
        help_text=_("Category applied to the event"),
        on_delete=models.CASCADE,
    )

    title_contains = models.CharField(
        verbose_name=_("Contained in the title"),
        help_text=_("Text contained in the event title"),
        max_length=512,
        blank=True,
        null=True,
    )
    title_exact = models.BooleanField(
        verbose_name=_("Exact title extract"),
        help_text=_(
            "If checked, the extract will be searched for in the title using the exact form (capitals, accents)."
        ),
        default=False,
    )

    description_contains = models.CharField(
        verbose_name=_("Contained in the description"),
        help_text=_("Text contained in the description"),
        max_length=512,
        blank=True,
        null=True,
    )
    desc_exact = models.BooleanField(
        verbose_name=_("Exact description extract"),
        help_text=_(
            "If checked, the extract will be searched for in the description using the exact form (capitals, accents)."
        ),
        default=False,
    )

    location_contains = models.CharField(
        verbose_name=_("Contained in the location"),
        help_text=_("Text contained in the event location"),
        max_length=512,
        blank=True,
        null=True,
    )
    loc_exact = models.BooleanField(
        verbose_name=_("Exact location extract"),
        help_text=_(
            "If checked, the extract will be searched for in the location using the exact form (capitals, accents)."
        ),
        default=False,
    )

    place = models.ForeignKey(
        Place,
        verbose_name=_("Place"),
        help_text=_("Location from place"),
        null=True,
        on_delete=models.SET_NULL,
        blank=True,
    )

    rules = None

    class Meta:
        verbose_name = _("Categorisation rule")
        verbose_name_plural = _("Categorisation rules")
        permissions = [("apply_categorisationrules", "Apply a categorisation rule")]

    # all rules are applied, starting from the first to the last
    def apply_rules(event):
        c = CategorisationRule.get_category_from_rules(event)
        
        if c is None:
            return 0
        else:
            event.category = c
            return 1

    def get_category_from_rules(event):
        cats = defaultdict(lambda: 0)
        if CategorisationRule.rules is None:
            CategorisationRule.rules = CategorisationRule.objects.all().select_related("category").select_related("place")

        for rule in CategorisationRule.rules:
            if rule.match(event):
                cats[rule.category] += rule.weight

        if len(cats) == 0:
            return None
        else:
            return max(cats, key=cats.get)

    def match(self, event):
        if self.description_contains and self.description_contains != "":
            if self.desc_exact:
                result = self.description_contains in event.description
            else:
                result = event.description and (
                    remove_accents(self.description_contains).lower()
                    in remove_accents(event.description).lower()
                )
            if not result:
                return False

        if self.title_contains and self.title_contains != "":
            if self.title_exact:
                result = self.title_contains in event.title
            else:
                result = (
                    remove_accents(self.title_contains).lower()
                    in remove_accents(event.title).lower()
                )
            if not result:
                return False

        if self.location_contains and self.location_contains != "":
            if self.loc_exact:
                result = self.location_contains in event.location
            else:
                result = (
                    remove_accents(self.location_contains).lower()
                    in remove_accents(event.location).lower()
                )
            if not result:
                return False

        if self.place:
            if not event.exact_location == self.place:
                return False


        return True


