import copy
import logging
import os
import urllib
import urllib.request
import uuid
from datetime import date, time, timedelta, datetime
from urllib.parse import urlparse

import emoji
import recurrence.fields
from django.contrib.auth.models import User
from django.contrib.gis.db.models.functions import Distance
from django.contrib.postgres.search import TrigramSimilarity
from django.contrib.sites.models import Site
from django.core.cache import cache
from django.core.cache.utils import make_template_fragment_key
from django.core.files import File
from django.core.files.storage import default_storage
from django.core.mail import send_mail
from django.db import models, connection
from django.db.models import Func, F, Q, Count
from django.db.models.expressions import RawSQL, Value, OuterRef, Subquery
from django.db.models.functions import Lower, ExtractDay, Now
from django.template.defaultfilters import date as _date, slugify
from django.template.loader import render_to_string
from django.urls import reverse
from django.utils import timezone
from django.utils.dateparse import parse_date
from django.utils.translation import gettext_lazy as _
from django_better_admin_arrayfield.models.fields import ArrayField
from django_resized import ResizedImageField
from icalendar import Calendar as icalCal
from icalendar import Event as icalEvent

from ..models.utils import remove_accents
from ..models.configuration import SiteConfiguration
from ..calendar import CalendarDay
from ..import_tasks.extractor import Extractor


logger = logging.getLogger(__name__)


class DuplicatedEvents(models.Model):
    representative = models.ForeignKey(
        "Event",
        verbose_name=_("Representative event"),
        help_text=_(
            "This event is the representative event of the duplicated events group"
        ),
        null=True,
        default=None,
        on_delete=models.SET_DEFAULT,
    )

    class Meta:
        verbose_name = _("Duplicated events")
        verbose_name_plural = _("Duplicated events")
        indexes = [
            models.Index(fields=["representative"]),
        ]

    def __init__(self, *args, **kwargs):
        self.events = None
        super().__init__(*args, **kwargs)

    def nb_duplicated(self):
        return self.event_set.count()

    def get_duplicated(self):
        if self.events is None:
            self.events = self.event_set.order_by("created_date").all()
        return self.events

    def get_absolute_url(self):
        return reverse("view_duplicate", kwargs={"pk": self.pk})

    def get_one_event(self):
        return self.representative

    def fixed(self):
        return self.representative is not None

    def is_published(self):
        return len([e for e in self.get_duplicated() if e.is_published()]) > 0

    def has_modified(self):
        return len([e for e in self.get_duplicated() if e.modified()]) > 0

    def has_local_version(self):
        return len([e for e in self.get_duplicated() if e.local_version()]) > 0

    def get_local_version(self):
        if self.representative and self.representative.local_version():
            return self.representative

        lv = [e for e in self.get_duplicated() if e.local_version()]
        if len(lv) == 0:
            return None
        else:
            lv.sort(key=lambda x: x.modified_date, reverse=True)
            return lv[0]

    def merge_into(self, other):
        # for all objects associated to this group
        for e in self.get_duplicated():
            # change their group membership
            e.other_versions = other
            # save them without updating modified date
            e.set_no_modification_date_changed()
            e.save()
        other.representative = None
        other.save()
        # then delete the empty group
        self.delete()

    # this method fixes the duplicated events by using the given event
    # as the representative one.
    # if no event is given, the last one (by creation date) is selected.
    def fix(self, event=None):
        events = self.get_duplicated()
        if event is None:
            events = events.order_by("-created_date")
        for e in events:
            if event is None:
                event = e
        if event is not None:
            event.status = Event.STATUS.PUBLISHED
            self.representative = event
        Event.objects.bulk_update(events, fields=["status"])
        self.save()
        return len(events)

    def merge_groups(groups):
        if len(groups) == 0:
            return None
        elif len(groups) == 1:
            return groups[0]
        else:
            result = groups[0]
            for g in groups[1:]:
                g.merge_into(result)
            return result

    def get_items_comparison(self):
        return Event.get_comparison(self.get_duplicated())

    @classmethod
    def remove_singletons(cls):
        singletons = DuplicatedEvents.objects.annotate(nb_events=Count("event")).filter(
            nb_events__lte=1
        )
        nb, d = singletons.delete()
        return nb

    def not_fixed_qs(qs=None, fixed=False):
        if not qs:
            qs = DuplicatedEvents.objects

        qs = qs.annotate(
            nb_no_trash=Count("event", filter=~Q(event__status=Event.STATUS.TRASH))
        )
        q = ~Q(representative__isnull=True) | Q(nb_no_trash__lte=1)
        if fixed:
            return qs.filter(q)
        else:
            return qs.exclude(q)

    def save(self, *args, **kwargs):
        if self.representative and self.representative not in self.event_set.all():
            self.representative = None

        super().save(*args, **kwargs)

    def get_import_messages(self):
        from . import Message

        msgs = []
        for e in self.get_duplicated():
            for m in e.message_set.order_by("date"):
                msgs.append(m)
        return msgs

    def get_event_index(self, event):
        for i, e in enumerate(self.get_duplicated()):
            if e.pk == event.pk:
                return i
        return -1


class Event(models.Model):
    class STATUS(models.TextChoices):
        PUBLISHED = "published", _("Published")
        DRAFT = "draft", _("Draft")
        TRASH = "trash", _("Trash")

    created_date = models.DateTimeField(editable=False)
    imported_date = models.DateTimeField(blank=True, null=True)
    modified_date = models.DateTimeField(blank=True, null=True)
    moderated_date = models.DateTimeField(blank=True, null=True)

    editing_start = models.DateTimeField(blank=True, null=True)
    editing_user = models.ForeignKey(
        User,
        verbose_name=_("Author currently editing/moderating the event"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="in_edition_events",
    )

    created_by_user = models.ForeignKey(
        User,
        verbose_name=_("Author of the event creation"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="created_events",
    )
    imported_by_user = models.ForeignKey(
        User,
        verbose_name=_("Author of the last importation"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="imported_events",
    )
    modified_by_user = models.ForeignKey(
        User,
        verbose_name=_("Author of the last modification"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="modified_events",
    )
    moderated_by_user = models.ForeignKey(
        User,
        verbose_name=_("Author of the last moderation"),
        null=True,
        blank=True,
        default=None,
        on_delete=models.SET_DEFAULT,
        related_name="moderated_events",
    )

    recurrence_dtstart = models.DateTimeField(editable=False, blank=True, null=True)
    recurrence_dtend = models.DateTimeField(editable=False, blank=True, null=True)

    title = models.CharField(verbose_name=_("Title"), max_length=512)

    status = models.CharField(
        _("Status"),
        max_length=20,
        choices=STATUS.choices,
        default=STATUS.DRAFT,
    )

    category = models.ForeignKey(
        "Category",
        verbose_name=_("Category"),
        null=True,
        default=None,
        on_delete=models.SET_DEFAULT,
    )

    start_day = models.DateField(verbose_name=_("Start day"))
    start_time = models.TimeField(
        verbose_name=_("Start time"),
        blank=True,
        null=True,
    )

    end_day = models.DateField(
        verbose_name=_("End day"),
        blank=True,
        null=True,
    )
    end_time = models.TimeField(verbose_name=_("End time"), blank=True, null=True)

    recurrences = recurrence.fields.RecurrenceField(
        verbose_name=_("Recurrence"),
        include_dtstart=False,
        blank=True,
        null=True,
    )

    exact_location = models.ForeignKey(
        "Place",
        verbose_name=_("Location"),
        null=True,
        on_delete=models.SET_NULL,
        blank=True,
    )
    location = models.CharField(
        verbose_name=_("Location (free form)"),
        help_text=_(
            "Address of the event in case its not available in the already known places (free form)"
        ),
        max_length=512,
        default="",
        null=True,
        blank=True,
    )

    local_event = models.BooleanField(
        verbose_name=_("Local event"),
        help_text=_(
            "If the event is a local event, it will not be proposed by default in daily, weekly or monthly views, unless the user has explicitly indicated that he wishes to see these events."
        ),
        default=False,
        null=False,
        blank=False,
    )

    description = models.TextField(
        verbose_name=_("Description"),
        blank=True,
        null=True,
    )

    organisers = models.ManyToManyField(
        "Organisation",
        related_name="organised_events",
        verbose_name=_("Organisers"),
        help_text=_(
            "list of event organisers. Organizers will only be displayed if one of them does not normally use the venue."
        ),
        blank=True,
    )

    local_image = ResizedImageField(
        size=[900, None],
        verbose_name=_("Illustration"),
        max_length=1024,
        blank=True,
        null=True,
    )

    image = models.URLField(
        verbose_name=_("Illustration (URL)"),
        help_text=_("External URL of the illustration image"),
        max_length=1024,
        blank=True,
        null=True,
    )
    image_alt = models.CharField(
        verbose_name=_("Illustration description"),
        help_text=_("Alternative text used by screen readers for the image"),
        blank=True,
        null=True,
        max_length=1024,
    )

    import_sources = ArrayField(
        models.CharField(max_length=512),
        verbose_name=_("Importation source"),
        help_text=_("Importation source used to detect removed entries."),
        blank=True,
        null=True,
    )
    uuids = ArrayField(
        models.CharField(max_length=512),
        verbose_name=_("UUIDs"),
        help_text=_("UUIDs from import to detect duplicated entries."),
        blank=True,
        null=True,
    )
    reference_urls = ArrayField(
        models.URLField(max_length=512),
        verbose_name=_("Online sources or ticketing"),
        blank=True,
        null=True,
    )

    tags = ArrayField(
        models.CharField(max_length=64),
        verbose_name=_("Tags"),
        blank=True,
        null=True,
    )

    other_versions = models.ForeignKey(
        DuplicatedEvents,
        verbose_name=_("Other versions"),
        on_delete=models.SET_NULL,
        null=True,
        blank=True,
    )

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.processing_user = None
        self._proposed_events = None
        self._messages = []

    def get_import_messages(self):
        return self.message_set.order_by("date")

    def get_consolidated_end_day(self, intuitive=True):
        if intuitive:
            end_day = self.get_consolidated_end_day(False)
            if end_day != self.start_day and self.end_time and self.end_time < time(8):
                if isinstance(end_day, str):
                    end_day = parse_date(end_day)
                return end_day + timedelta(days=-1)
            else:
                return end_day
        else:
            return self.end_day if self.end_day else self.start_day

    def is_modification_locked(self, now=None):
        if now is None:
            now = timezone.now()
        limit = timezone.now() + timedelta(minutes=-10)

        return self.editing_start is not None and self.editing_start > limit

    def get_modification_lock(self, user):
        now = timezone.now()

        if not self.is_modification_locked(now):
            self.editing_start = now
            self.editing_user = user
            self.save(update_fields=["editing_start", "editing_user"])
            return True
        else:
            return False

    def free_modification_lock(self, user, save=True):
        if user != self.editing_user:
            return False
        else:
            self.editing_start = None
            self.editing_user = None
            if save:
                self.save(update_fields=["editing_start", "editing_user"])
            return True

    def get_dates(self):
        first = self.start_day
        last = self.get_consolidated_end_day()
        return [first + timedelta(n) for n in range(int((last - first).days) + 1)]

    def is_single_day(self, intuitive=True):
        return self.start_day == self.get_consolidated_end_day(intuitive)

    def is_event_long_duration(self, min_days=4, intuitive=True):
        end_date = self.get_consolidated_end_day(intuitive)
        if isinstance(end_date, str):
            end_date = parse_date(end_date)
        return parse_date(self.start_day) + timedelta(days=min_days) < end_date

    def add_message(self, msg):
        self._messages.append(msg)

    def get_messages(self):
        return self._messages

    def has_message(self):
        return len(self._messages) > 0

    def contains_date(self, d, intuitive=True):
        return d >= self.start_day and d <= self.get_consolidated_end_day(intuitive)

    def get_absolute_url(self):
        return reverse(
            "view_event",
            kwargs={
                "year": self.start_day.year,
                "month": self.start_day.month,
                "day": self.start_day.day,
                "pk": self.pk,
                "extra": slugify(self.title),
            },
        )

    def in_recurrent_import(self):
        from . import RecurrentImport

        if self.import_sources:
            for s in self.import_sources:
                o_s = RecurrentImport.objects.filter(source=s).count()
                if o_s > 0:
                    return True
        return False

    def get_import_sources(self):
        from . import RecurrentImport

        if self.import_sources:
            result = []
            for s in self.import_sources:
                o_s = RecurrentImport.objects.filter(source=s).first()
                if o_s:
                    result.append((o_s.get_absolute_url(), o_s.name))
                else:
                    result.append((s, s))
            return result
        else:
            return []

    def __str__(self):
        return _date(self.start_day) + ": " + self.title

    class Meta:
        verbose_name = _("Event")
        verbose_name_plural = _("Events")
        permissions = [("set_duplicated_event", "Can set an event as duplicated")]
        indexes = [
            models.Index(fields=["start_day", "start_time"]),
            models.Index(fields=["end_day", "end_time"]),
            models.Index(fields=["status"]),
            models.Index(fields=["recurrences"]),
            models.Index(fields=["recurrence_dtstart", "recurrence_dtend"]),
            models.Index("start_time", Lower("title"), name="start_time title"),
            models.Index(
                "start_time",
                "start_day",
                "end_day",
                "end_time",
                Lower("title"),
                name="datetimes title",
            ),
        ]

    def alternative_events_if_not_found(day, title):
        qs = (
            Event.objects.filter(
                Q(other_versions__isnull=True)
                | Q(other_versions__representative=F("pk"))
                | Q(other_versions__representative__isnull=True)
            )
            .filter(status=Event.STATUS.PUBLISHED)
            .annotate(similarity_title=TrigramSimilarity("title", title))
            .annotate(
                nbday_distance=Func((ExtractDay(F("start_day") - day)), function="ABS")
            )
            .annotate(
                score=F("similarity_title") * 200 + 30 / (F("nbday_distance") + 1)
            )
            .order_by("-score")[:10]
        )

        return qs.only(
            "title", "start_day", "start_time", "exact_location", "location", "tags"
        )

    def proposed_events(self):
        threshold_distance = 30000
        min_tags = 1

        if self._proposed_events is None:
            qs = (
                Event.objects.filter(
                    Q(start_day__gte=Now()) & Q(category=self.category) & ~Q(pk=self.pk)
                )
                .filter(
                    Q(other_versions__isnull=True)
                    | Q(other_versions__representative=F("pk"))
                    | Q(other_versions__representative__isnull=True)
                )
                .filter(status=Event.STATUS.PUBLISHED)
            )

            if self.tags and len(self.tags) > 0:
                qs = qs.annotate(
                    overlap_tags=RawSQL(
                        sql="ARRAY(select UNNEST(%s::text[]) INTERSECT select UNNEST(tags))",
                        params=(self.tags,),
                        output_field=ArrayField(models.CharField(max_length=50)),
                    )
                ).annotate(
                    overlap_tags_count=Func(
                        F("overlap_tags"),
                        function="CARDINALITY",
                        output_field=models.IntegerField(),
                    )
                )
                with_emoji = [
                    s for s in self.tags if any(char in emoji.EMOJI_DATA for char in s)
                ]
                if len(with_emoji) > 0:
                    qs = qs.annotate(
                        overlap_emoji_tags=RawSQL(
                            sql="ARRAY(select UNNEST(%s::text[]) INTERSECT select UNNEST(tags))",
                            params=(with_emoji,),
                            output_field=ArrayField(models.CharField(max_length=50)),
                        )
                    ).annotate(
                        overlap_emoji_tags_count=Func(
                            F("overlap_emoji_tags"),
                            function="CARDINALITY",
                            output_field=models.IntegerField(),
                        )
                    )
                else:
                    qs = qs.annotate(overlap_emoji_tags_count=Value(0))

            else:
                qs = qs.annotate(
                    overlap_tags_count=Value(0), overlap_emoji_tags_count=Value(0)
                )

            if self.exact_location:
                qs = (
                    qs.filter(overlap_tags_count__gte=min_tags)
                    .annotate(
                        distance=Distance(
                            F("exact_location__location"), self.exact_location.location
                        )
                    )
                    .filter(distance__lte=threshold_distance)
                )
            else:
                qs = qs.annotate(distance=Value(1))

            qs = (
                qs.annotate(
                    nbday_distance=Func(
                        (ExtractDay(F("start_day") - self.start_day)), function="ABS"
                    )
                )
                .annotate(similarity_title=TrigramSimilarity("title", self.title))
                .annotate(
                    similarity_description=TrigramSimilarity(
                        "description", self.description
                    )
                )
                .annotate(
                    score=F("overlap_tags_count") * 20
                    + F("overlap_emoji_tags_count") * 40
                    + F("similarity_title") * 2
                    + F("similarity_description") * 10
                    + 10 / (F("distance") + 1)
                    + 30 / (F("nbday_distance") + 1)
                )
                .order_by("-score")[:10]
            )

            self._proposed_events = qs.only(
                "title", "start_day", "start_time", "exact_location", "location", "tags"
            )
            self._proposed_events = sorted(
                self._proposed_events,
                key=lambda e: (
                    e.start_day,
                    e.start_time if e.start_time else datetime.min.time(),
                    e.title,
                ),
            )

        return self._proposed_events

    def chronology_dates(self):
        return self.chronology(True)

    def chronology(self, simple=False):
        c = []
        if self.modified_date:
            c.append(
                {
                    "timestamp": self.modified_date,
                    "data": "modified_date",
                    "user": self.modified_by_user,
                    "is_date": True,
                }
            )
        if self.moderated_date:
            c.append(
                {
                    "timestamp": self.moderated_date,
                    "data": "moderated_date",
                    "user": self.moderated_by_user,
                    "is_date": True,
                }
            )
        if self.imported_date:
            c.append(
                {
                    "timestamp": self.imported_date,
                    "data": "imported_date",
                    "user": self.imported_by_user,
                    "is_date": True,
                }
            )
        if self.created_date:
            if self.created_by_user:
                user = self.created_by_user
            else:
                if self.in_recurrent_import():
                    user = _("recurrent import")
                else:
                    user = _("a non authenticated user")

            c.append(
                {
                    "timestamp": self.created_date + timedelta(milliseconds=-1),
                    "data": "created_date",
                    "user": user,
                    "is_date": True,
                }
            )

        if not simple:
            c += [
                {
                    "timestamp": m.date,
                    "data": m,
                    "user": m.user,
                    "is_date": False,
                }
                for m in self.message_set.filter(spam=False)
            ]

            if self.other_versions:
                for o in self.other_versions.get_duplicated():
                    if o != self:
                        c += [
                            {
                                "timestamp": m.date,
                                "data": m,
                                "user": m.user,
                                "is_date": False,
                            }
                            for m in o.message_set.filter(spam=False)
                        ]

        c.sort(key=lambda x: x["timestamp"])

        return c

    def sorted_tags(self):
        if self.tags is None:
            return []
        else:
            result = self.tags
            result.sort(
                key=lambda x: emoji.demojize(
                    remove_accents(x.lower()), delimiters=("000", "")
                )
            )
            return result

    def tag_messages(self):
        from ..models.tag import Tag

        if self.tags is None:
            return []
        return Tag.objects.filter(
            Q(name__in=self.tags)
            & Q(message__isnull=False)
            & ~Q(message__in=["", "<p></p>", "<p>&nbsp;</p>"])
        ).values("message")

    def get_all_tags(sort=True):
        cursor = connection.cursor()
        raw_query = """
        select unnest(subquery_alias.tags) as distinct_tags, count(*) as tags_group_by_count
            from (select tags from agenda_culturel_event) as subquery_alias group by distinct_tags;
        """
        cursor.execute(raw_query)
        result = [{"tag": row[0], "count": row[1]} for row in cursor]
        if sort:
            result.sort(
                key=lambda x: emoji.demojize(
                    remove_accents(x["tag"].lower()), delimiters=("000", "")
                )
            )
        return result

    def is_draft(self):
        return self.status == Event.STATUS.DRAFT

    def is_published(self):
        return self.status == Event.STATUS.PUBLISHED

    def is_trash(self):
        return self.status == Event.STATUS.TRASH

    def modified(self):
        return (
            not self.pure_import()
            and (self.modified_date - self.created_date).total_seconds() > 1
        )

    def pure_import(self):
        if self.imported_date is None:
            return False
        return (
            self.modified_date is None
            or (self.modified_date - self.imported_date).total_seconds() <= 0
        )

    def local_version(self):
        return self.imported_date is None or self.modified()

    def get_reference_urls(self):
        res = [] if self.reference_urls is None else copy.copy(self.reference_urls)

        if self.other_versions:
            for o in self.other_versions.get_duplicated():
                if o.status == Event.STATUS.PUBLISHED and o.reference_urls is not None:
                    res += o.reference_urls

        res = list(set(res))
        res.sort()

        return res

    def get_local_version(self):
        # a non-pure import is a local version
        if not self.pure_import():
            return self

        # in case of non other version, return None
        if self.other_versions is None:
            return None

        return self.other_versions.get_local_version()

    def get_shown_organisers(self):
        if self.organisers.count() == 0:
            return None
        if self.exact_location is None:
            has_significant = True
        else:
            has_significant = (
                self.organisers.filter(~Q(principal_place=self.exact_location)).count()
                > 0
            )

        if has_significant:
            return self.organisers.all()
        else:
            return None

    def get_nb_not_moderated(first_day, nb_mod_days=21, nb_classes=4):
        window_end = first_day + timedelta(days=nb_mod_days)
        nb_not_moderated = (
            Event.objects.filter(~Q(status=Event.STATUS.TRASH))
            .filter(Q(start_day__gte=first_day) & Q(start_day__lte=window_end))
            .filter(
                Q(other_versions__isnull=True)
                | Q(other_versions__representative=F("pk"))
                | Q(other_versions__representative__isnull=True)
            )
            .values("start_day")
            .annotate(
                not_moderated=Count("start_day", filter=Q(moderated_date__isnull=True))
            )
            .annotate(nb_events=Count("start_day"))
            .order_by("start_day")
            .values("not_moderated", "nb_events", "start_day")
        )

        max_not_moderated = max([x["not_moderated"] for x in nb_not_moderated])
        if max_not_moderated == 0:
            max_not_moderated = 1
        nb_not_moderated_dict = dict(
            [
                (x["start_day"], (x["not_moderated"], x["nb_events"]))
                for x in nb_not_moderated
            ]
        )
        # add missing dates
        date_list = [first_day + timedelta(days=x) for x in range(0, nb_mod_days)]
        nb_not_moderated = [
            {
                "start_day": d,
                "is_today": d == first_day,
                "nb_events": (
                    nb_not_moderated_dict[d][1] if d in nb_not_moderated_dict else 0
                ),
                "not_moderated": (
                    nb_not_moderated_dict[d][0] if d in nb_not_moderated_dict else 0
                ),
            }
            for d in date_list
        ]
        nb_not_moderated = [
            x
            | {
                "note": (
                    0
                    if x["not_moderated"] == 0
                    else int((nb_classes - 1) * x["not_moderated"] / max_not_moderated)
                    + 1
                )
            }
            for x in nb_not_moderated
        ]
        return [nb_not_moderated[x : x + 7] for x in range(0, len(nb_not_moderated), 7)]

    def is_know_url_get_visible_event(url, auth):
        evs = Event.objects.filter(uuids__contains=[url])
        if auth:
            e = evs.first()
            if e is not None:
                if e.is_representative():
                    return e
                else:
                    return e.other_versions.representative
            else:
                return False
        else:
            e_all = evs.exists()
            if not e_all:
                return False
            else:
                e = evs.first()
                if e is not None:
                    if not e.is_representative():
                        e = e.other_versions.representative
                    return e if e.status == Event.STATUS.PUBLISHED else True
                else:
                    return False

    @classmethod
    def nb_draft_events(cls):
        return Event.objects.filter(status=Event.STATUS.DRAFT).count()

    @classmethod
    def get_qs_events_with_unkwnon_place(cls):
        return (
            Event.objects.filter(exact_location__isnull=True)
            .filter(~Q(status=Event.STATUS.TRASH))
            .filter(Q(other_versions=None) | Q(other_versions__representative=F("pk")))
        )

    def is_representative(self):
        return self.other_versions is None or self.other_versions.representative == self

    def download_missing_image(self):
        if self.local_image and not default_storage.exists(self.local_image.name):
            self.download_image()
            self.save(update_fields=["local_image"])

    def download_image(self):
        # first download file

        if str(self.image) != "":
            a = urlparse(self.image)
            basename = os.path.basename(a.path)

            try:
                basename = basename.decode()
            except Exception:
                pass

            ext = basename.split(".")[-1]
            if ext[:3].lower() != "svg":  # temporary hack to ignore svg files
                filename = "%s.%s" % (uuid.uuid4(), ext)

                try:
                    tmpfile, _ = urllib.request.urlretrieve(self.image)
                except Exception:
                    return None

                # if the download is ok, then create the corresponding file object
                self.local_image = File(name=filename, file=open(tmpfile, "rb"))

    def add_pending_organisers(self, organisers):
        self.pending_organisers = organisers

    def has_pending_organisers(self):
        return hasattr(self, "pending_organisers")

    def set_is_not_found_import(self):
        self.not_found_import = True

    def is_not_found_import(self):
        return hasattr(self, "not_found_import")

    def set_skip_duplicate_check(self):
        self.skip_duplicate_check = True

    def is_skip_duplicate_check(self):
        return hasattr(self, "skip_duplicate_check")

    def is_in_importation_process(self):
        return hasattr(self, "in_importation_process")

    def set_in_importation_process(self):
        self.in_importation_process = True

    def is_no_modification_date_changed(self):
        return hasattr(self, "no_modification_date_changed")

    def set_no_modification_date_changed(self):
        self.no_modification_date_changed = True

    def set_processing_user(self, user):
        if user is None or user.is_anonymous:
            self.processing_user = None
        else:
            self.processing_user = user

    def set_in_moderation_process(self):
        self.in_moderation_process = True

    def is_in_moderation_process(self):
        return hasattr(self, "in_moderation_process")

    def set_invalid_start_time(self):
        self.invalid_start_time = True

    def has_invalid_start_time(self):
        return hasattr(self, "invalid_start_time")

    def set_invalid_start_date(self):
        self.invalid_start_date = True

    def has_invalid_start_date(self):
        return hasattr(self, "invalid_start_date")

    def set_invalid_title(self):
        self.invalid_title = True

    def has_invalid_title(self):
        return hasattr(self, "invalid_title")

    def update_modification_dates(self):
        now = timezone.now()
        if not self.id:
            self.created_date = now
            self.created_by_user = self.processing_user
        if self.is_in_importation_process():
            self.imported_date = now
            self.imported_by_user = self.processing_user
        if self.modified_date is None or not self.is_no_modification_date_changed():
            self.modified_date = now
            self.modified_by_user = self.processing_user
        if self.is_in_moderation_process():
            self.moderated_date = now
            self.moderated_by_user = self.processing_user

        # release editing lock
        if self.processing_user is not None:
            self.free_modification_lock(self.processing_user, False)

    def get_recurrence_at_date(self, year, month, day):
        dtstart = timezone.make_aware(
            datetime(year, month, day, 0, 0), timezone.get_default_timezone()
        )
        recurrences = self.get_recurrences_between(dtstart, dtstart)
        if len(recurrences) == 0:
            return self
        else:
            return recurrences[0]

    def get_image_url(self, request=None):
        if self.local_image and hasattr(self.local_image, "url"):
            try:
                if request:
                    return request.build_absolute_uri(self.local_image.url)
                else:
                    return self.local_image.url
            except Exception:
                pass
        if self.image:
            return self.image
        else:
            return None

    def has_image_url(self):
        return self.get_image_url() is not None

    # return a copy of the current object for each recurrence between first an last date (included)
    def get_recurrences_between(self, firstdate, lastdate):
        if not self.has_recurrences():
            return [self]
        else:
            result = []
            dtstart = timezone.make_aware(
                datetime.combine(self.start_day, time()),
                timezone.get_default_timezone(),
            )
            self.recurrences.dtstart = dtstart
            for d in self.recurrences.between(
                firstdate, lastdate, inc=True, dtstart=dtstart
            ):
                c = copy.deepcopy(self)
                c.start_day = d.date()
                if c.end_day is not None:
                    shift = d.date() - self.start_day
                    c.end_day += shift
                result.append(c)

            return result

    def has_recurrences(self):
        return self.recurrences is not None and (
            len(self.recurrences.rrules) != 0 or len(self.recurrences.rdates) != 0
        )

    def update_recurrence_dtstartend(self):
        # get start and end date and time
        sday = (
            date.fromisoformat(self.start_day)
            if isinstance(self.start_day, str)
            else self.start_day
        )
        eday = (
            date.fromisoformat(self.end_day)
            if isinstance(self.end_day, str)
            else self.end_day
        )
        stime = (
            time.fromisoformat(self.start_time)
            if isinstance(self.start_time, str)
            else time() if self.start_time is None else self.start_time
        )
        etime = (
            time.fromisoformat(self.end_time)
            if isinstance(self.end_time, str)
            else time() if self.end_time is None else self.end_time
        )

        # updating start and end date for database efficiency
        self.recurrence_dtstart = datetime.combine(sday, stime)
        if not self.has_recurrences():
            if self.end_day is None:
                self.dtend = None
            else:
                self.recurrence_dtend = datetime.combine(eday, etime)
        else:
            if (
                len(self.recurrences.rrules) > 0
                and self.recurrences.rrules[0].until is None
                and self.recurrences.rrules[0].count is None
            ):
                self.recurrence_dtend = None
            else:
                self.recurrences.dtstart = datetime.combine(sday, time())
                occurrence = self.recurrences.occurrences()
                try:
                    self.recurrence_dtend = occurrence[-1]
                    if (
                        self.recurrences.dtend is not None
                        and self.recurrences.dtstart is not None
                    ):
                        self.recurrence_dtend += (
                            self.recurrences.dtend - self.recurrences.dtstart
                        )
                except Exception:
                    self.recurrence_dtend = self.recurrence_dtstart

    def prepare_save(self):
        from ..models.place import Place
        from ..models.category import Category, CategorisationRule

        self.update_modification_dates()
        self.update_recurrence_dtstartend()

        # clear cache
        for is_auth in [False, True]:
            key = make_template_fragment_key("event_body", [is_auth, self.pk])
            cache.delete(key)

        # if the image is defined but not locally downloaded
        if self.image and (
            not self.local_image or not default_storage.exists(self.local_image.name)
        ):
            self.download_image()

        if self.tags:
            # remove "/" from tags
            self.tags = [t.replace("/", "-") for t in self.tags]
            # remove duplicate tags
            self.tags = list(set(self.tags))

        # in case of importation process
        if self.is_in_importation_process():
            # try to detect location
            if not self.exact_location:
                for p in Place.objects.all():
                    if p.match(self):
                        self.exact_location = p
                        break
            # try to detect category
            if not self.category or self.category.name == Category.default_name:
                CategorisationRule.apply_rules(self)

    def get_contributor_message(self):
        from . import Message

        types = [
            Message.TYPE.FROM_CONTRIBUTOR,
            Message.TYPE.FROM_CONTRIBUTOR_NO_MSG,
        ]
        if self.other_versions is None or self.other_versions.representative is None:
            return Message.objects.filter(
                related_event=self.pk, message_type__in=types, closed=False
            )
        else:
            return Message.objects.filter(
                related_event__in=self.other_versions.get_duplicated(),
                message_type__in=types,
                closed=False,
            )

    def notify_if_required(self, request):
        notif = False
        if self.status != Event.STATUS.DRAFT:
            messages = self.get_contributor_message()
            if messages:
                for message in messages:
                    if (
                        message
                        and not message.closed
                        and message.email
                        and message.email != ""
                    ):
                        # send email
                        context = {
                            "sitename": Site.objects.get_current(request).name,
                            "event_title": self.title,
                        }
                        if self.status == Event.STATUS.PUBLISHED:
                            context["url"] = request.build_absolute_uri(
                                self.get_absolute_url()
                            )
                            subject = _("Your event has been published")
                            body = render_to_string(
                                "agenda_culturel/emails/published.txt", context
                            )
                        else:
                            subject = _("Your message has not been retained")
                            body = render_to_string(
                                "agenda_culturel/emails/retained.txt", context
                            )

                        send_mail(subject, body, None, [message.email])
                        message.closed = True
                        message.save()
                        notif = True

        return notif

    def save(self, *args, **kwargs):
        self.prepare_save()

        # check for similar events if no duplicated is known and only if the event is created
        if (
            self.pk is None
            and self.other_versions is None
            and not self.is_skip_duplicate_check()
        ):
            # and if this is not an importation process
            if not self.is_in_importation_process():
                similar_events = self.find_similar_events()

                # if it exists similar events, add this relation to the event
                if len(similar_events) != 0:
                    self.set_other_versions(similar_events)

        # check if it's a clone (that will become representative)
        clone = self.pk is None and self.other_versions is not None

        # check if we need to clean the other_versions
        if (
            not clone
            and "noclean_other_versions" not in kwargs
            and self.pk
            and self.other_versions is not None
            and self.other_versions.nb_duplicated() == 1
        ):
            self.other_versions.delete()
            self.other_versions = None
        if "noclean_other_versions" in kwargs:
            del kwargs["noclean_other_versions"]

        # first save the current object
        super().save(*args, **kwargs)

        # notify only if required (and request is known)
        if "request" in kwargs:
            self.notify_if_required(kwargs.get("request"))

        # save message if required
        if self.has_message():
            for msg in self.get_messages():
                msg.related_event = self
                msg.save()

        # then if its a clone, update the representative
        if clone:
            self.other_versions.representative = self
            self.other_versions.save()
            # if we just clone a single event, its status is
            # the same as the status of the current object
            if self.other_versions.get_duplicated().count() == 2:
                for e in self.other_versions.get_duplicated():
                    if e != self:
                        e.status = self.status
                        e.set_no_modification_date_changed()
                        e.save()

    def find_similar_events(self):
        start_time_test = Q(start_time=self.start_time)

        if self.start_time is not None:
            # convert str start_time to time
            if isinstance(self.start_time, str):
                self.start_time = time.fromisoformat(self.start_time)
            interval = (
                (
                    time(self.start_time.hour - 1, self.start_time.minute)
                    if self.start_time.hour >= 1
                    else time(0, 0)
                ),
                (
                    time(self.start_time.hour + 1, self.start_time.minute)
                    if self.start_time.hour < 23
                    else time(23, 59)
                ),
            )
            start_time_test = start_time_test | Q(start_time__range=interval)

        return (
            Event.objects.annotate(
                similarity_title=TrigramSimilarity("title", self.title)
            )
            .annotate(similarity_location=TrigramSimilarity("location", self.location))
            .filter(
                Q(start_day=self.start_day)
                & start_time_test
                & Q(similarity_title__gt=0.5)
                & Q(similarity_title__gt=0.3)
            )
        )

    def find_same_events_by_uuid(self):
        return (
            None
            if self.uuids is None or len(self.uuids) == 0
            else Event.objects.filter(uuids__contains=self.uuids)
        )

    def get_updateable_uuid(self):
        if self.uuids and len(self.uuids) > 0:
            for s in self.uuids:
                if Extractor.is_known_url_default_extractors(s):
                    return s
        return None

    def is_updateable(self):
        return self.get_updateable_uuid() is not None

    def split_uuid(uuid):
        els = uuid.split(":")
        if len(els) == 1:
            return ":".join(els[0:-1]), 0
        else:
            if els[-1].isdigit():
                return ":".join(els[0:-1]), int(els[-1])
            else:
                return ":".join(els), 0

    def is_ancestor_uuid_and_rules(uuid1, uuid2, r1, r2):
        root1, version1 = Event.split_uuid(uuid1)
        root2, version2 = Event.split_uuid(uuid2)
        return root1 == root2 and ((r1 and not r2) or (version1 < version2))

    def is_ancestor_by_uuid_and_rules(self, event):
        if self.uuids is None or event.uuids is None:
            return False
        if (not self.recurrences) and event.recurrences:
            return False

        for s_uuid in self.uuids:
            for e_uuid in event.uuids:
                if Event.is_ancestor_uuid_and_rules(
                    s_uuid, e_uuid, self.recurrences, event.recurrences
                ):
                    return True
        return False

    def same_uuid(self, event):
        if self.uuids is None or event.uuids is None:
            return False
        for s_uuid in self.uuids:
            for e_uuid in event.uuids:
                if s_uuid == e_uuid:
                    return True
        return False

    def get_other_not_trash_versions(self):
        if self.other_versions is None:
            return []
        else:
            return [
                e
                for e in self.other_versions.get_duplicated()
                if e.pk != self.pk and e.status != Event.STATUS.TRASH
            ]

    def get_other_versions(self):
        if self.other_versions is None:
            return []
        else:
            return [e for e in self.other_versions.get_duplicated() if e.pk != self.pk]

    def masked(self):
        return (
            self.other_versions
            and self.other_versions.representative is not None
            and self.other_versions.representative.pk != self.pk
        )

    def get_organisers(self):
        if self.pk:
            if self.organisers is None:
                return []
            else:
                return self.organisers.all()
        else:
            if self.has_pending_organisers():
                if self.pending_organisers is None:
                    return []
                else:
                    return self.pending_organisers
            else:
                return []

    def get_comparison(events, all=True):
        result = []
        for attr in Event.data_fields(all=all, local_img=False, exact_location=False):
            if attr == "organisers":
                values = [[str(o) for o in e.get_organisers()] for e in events]
            else:
                values = [getattr(e, attr) for e in events]
            values = ["" if v is None else v for v in values]
            values = [[] if attr == "tags" and v == "" else v for v in values]
            # only consider fixed part of Facebook urls
            if attr == "image":
                values = [v.split("?")[0] if "fbcdn.net" in v else v for v in values]

            if attr == "description":
                values = [v.replace("\r\n", "\n") for v in values]
            if len(set([str(v).strip() for v in values])) == 1:
                result.append({"similar": True, "key": attr, "values": values[0]})
            else:
                result.append({"similar": False, "key": attr, "values": values})
        return result

    def similar(self, event, all=True):
        res = Event.get_comparison([self, event], all)
        for r in res:
            if not r["similar"]:
                if r["key"] == "title" and (
                    self.has_invalid_title() or event.has_invalid_title()
                ):
                    continue
                if r["key"] == "start_day" and (
                    self.has_invalid_start_date() or event.has_invalid_start_date()
                ):
                    continue
                if r["key"] == "start_time" and (
                    self.has_invalid_start_time() or event.has_invalid_start_time()
                ):
                    continue
                # only consider path for Facebook urls
                if (
                    r["key"] == "image"
                    and len(r["values"]) > 0
                    and "fbcdn.net" in r["values"][0]
                ):
                    nb = len(set([urlparse(url).path for url in r["values"]]))
                    if nb == 1:
                        continue

                return False
        return True

    def set_other_versions(self, events, force_non_fixed=False):
        # get existing groups
        groups = list(set([e.other_versions for e in events] + [self.other_versions]))
        groups = [g for g in groups if g is not None]

        # do we have to create a new group?
        if len(groups) == 0:
            group = DuplicatedEvents.objects.create()
        else:
            # otherwise merge existing groups
            group = DuplicatedEvents.merge_groups(groups)

        group.save()

        if force_non_fixed:
            group.representative = None

        # set the possibly duplicated group for the current object
        self.other_versions = group

        # and for the other events
        for e in events:
            e.other_versions = group

        # finally update all events (including current if already created)
        elist = list(events) + ([self] if self.pk is not None else [])
        Event.objects.bulk_update(elist, fields=["other_versions"])

    def data_fields(local_img=True, exact_location=True, all=True, no_m2m=False):
        result = []

        if all:
            result += [
                "category",
                "tags",
            ]
            if not no_m2m:
                result += ["organisers"]

        result += [
            "title",
            "location",
            "start_day",
            "start_time",
            "end_day",
            "end_time",
            "description",
            "image",
        ]
        if all and local_img:
            result += ["local_image"]
        if all and exact_location:
            result += ["exact_location"]
        result += ["image_alt", "reference_urls", "recurrences"]

        return result

    def find_last_imported(events):
        events = [e for e in events if e.imported_date is not None]
        if len(events) == 0:
            return None
        else:
            events.sort(key=lambda e: e.imported_date, reverse=True)
            return events[0]

    def find_last_pure_import(events):
        events = [e for e in events if e.pure_import()]
        if len(events) == 0:
            return None
        else:
            events.sort(key=lambda e: e.imported_date, reverse=True)
            return events[0]

    def update(self, other, all):
        # integrate pending organisers
        if other.has_pending_organisers() and other.pending_organisers is not None:
            self.organisers.set(other.pending_organisers)

        logger.info(
            "process update "
            + other.title
            + " "
            + str(other.has_invalid_start_date())
            + " "
            + str(other.has_invalid_start_time())
        )
        # set attributes
        for attr in Event.data_fields(all=all, no_m2m=True):
            if attr == "title" and other.has_invalid_title():
                continue
            if attr == "start_day" and other.has_invalid_start_date():
                continue
            if attr == "start_time" and other.has_invalid_start_time():
                continue
            if attr == "exact_location" and other.exact_location is None:
                continue
            setattr(self, attr, getattr(other, attr))

        # adjust modified date if required
        if other.modified_date and self.modified_date < other.modified_date:
            self.modified_date = other.modified_date

        # add a possible missing uuid
        if self.uuids is None:
            self.uuids = []
        for uuide in other.uuids:
            if uuide not in self.uuids:
                self.uuids.append(uuide)

        # add possible missing sources
        if other.import_sources:
            if not self.import_sources:
                self.import_sources = []
            for source in other.import_sources:
                if source not in self.import_sources:
                    self.import_sources.append(source)

    def set_current_date(self, date):
        self.current_date = date

    def get_start_end_datetimes(self, day):
        time().min
        if self.start_day == day:
            if self.start_time is None:
                dtstart = datetime.combine(self.start_day, time().min)
            else:
                dtstart = datetime.combine(self.start_day, self.start_time)
        else:
            dtstart = datetime.combine(day, time().min)

        end_day = self.get_consolidated_end_day()

        if end_day == day:
            if self.end_time is None:
                if end_day == self.start_day:
                    dtend = dtstart + timedelta(hours=2)
                else:
                    dtend = datetime.combine(end_day, time().max)
            else:
                dtend = datetime.combine(end_day, self.end_time)
        else:
            dtend = datetime.combine(day, time().max)

        return dtstart, dtend

    def get_concurrent_events(self, remove_same_dup=True):
        day = self.current_date if hasattr(self, "current_date") else self.start_day
        day_events = CalendarDay(
            day,
            qs=Event.objects.filter(status=Event.STATUS.PUBLISHED).only(
                "start_day",
                "start_time",
                "title",
                "category",
                "other_versions",
                "recurrences",
                "end_day",
                "end_time",
                "uuids",
            ),
        ).get_events()
        return [
            e
            for e in day_events
            if e != self
            and self.is_concurrent_event(e, day)
            and (e.other_versions is None or e.other_versions != self.other_versions)
        ]

    def is_concurrent_event(self, e, day):
        dtstart, dtend = self.get_start_end_datetimes(day)
        e_dtstart, e_dtend = e.get_start_end_datetimes(day)

        return (dtstart <= e_dtstart <= dtend) or (e_dtstart <= dtstart <= e_dtend)

    def export_to_ics(events, request):
        cal = icalCal()
        # Some properties are required to be compliant
        cal.add(
            "prodid",
            "-//"
            + SiteConfiguration.get_solo().site_name
            + "//"
            + SiteConfiguration.get_solo().site_url
            + "//",
        )
        cal.add("version", "2.0")

        for event in events:
            event.get_consolidated_end_day()
            eventIcal = icalEvent()
            # mapping
            if event.start_time is None:
                eventIcal.add(
                    "dtstart",
                    date(
                        event.start_day.year,
                        event.start_day.month,
                        event.start_day.day,
                    ),
                )
            else:
                eventIcal.add(
                    "dtstart",
                    datetime(
                        event.start_day.year,
                        event.start_day.month,
                        event.start_day.day,
                        event.start_time.hour,
                        event.start_time.minute,
                    ),
                )
            if event.end_day is not None:
                if event.end_time is None:
                    eventIcal.add(
                        "dtend",
                        date(
                            event.end_day.year,
                            event.end_day.month,
                            event.end_day.day,
                        ),
                    )
                else:
                    eventIcal.add(
                        "dtend",
                        datetime(
                            event.end_day.year,
                            event.end_day.month,
                            event.end_day.day,
                            event.end_time.hour,
                            event.end_time.minute,
                        ),
                    )
            eventIcal.add("summary", event.title)
            eventIcal.add("name", event.title)
            url = (
                ("\n" + event.reference_urls[0])
                if event.reference_urls and len(event.reference_urls) > 0
                else ""
            )
            description = event.description if event.description else ""
            eventIcal.add("description", description + url)
            if event.local_image is not None and event.local_image != "":
                eventIcal.add(
                    "image",
                    request.build_absolute_uri(event.local_image),
                    parameters={"VALUE": "URI"},
                )
            eventIcal.add("location", event.exact_location or event.location)

            cal.add_component(eventIcal)

        return cal

    def get_count_modification(when):
        from . import RecurrentImport

        start = datetime(when[0].year, when[0].month, when[0].day)
        end = start + timedelta(days=when[1])

        recurrentimport = (
            RecurrentImport.objects.filter(source=OuterRef("import_sources__0"))
            .order_by()
            .annotate(count=Func(F("id"), function="Count"))
            .values("count")
        )

        nb_manual_creation = (
            Event.objects.filter(
                created_date__gte=start,
                created_date__lt=end,
                imported_date__isnull=True,
            )
            .filter(Q(uuids__len=0) | Q(uuids=None))
            .count()
        )
        nb_local_copies = Event.objects.filter(
            created_date__gte=start,
            created_date__lt=end,
            imported_date__isnull=True,
            uuids__len__gt=0,
        ).count()

        imported = Event.objects.filter(
            created_date__gte=start,
            created_date__lt=end,
            imported_date__isnull=False,
            uuids__len__gt=0,
        ).annotate(nb_rimport=Subquery(recurrentimport))

        nb_manual_import = imported.filter(
            Q(import_sources__len=0) | Q(nb_rimport=0)
        ).count()
        nb_first_import = imported.filter(nb_rimport__gt=0).count()

        nb_moderated = Event.objects.filter(
            moderated_date__gte=start, moderated_date__lt=end
        ).count()

        return {
            "when": (start, start + timedelta(days=when[1] - 1)),
            "nb_manual_creation": nb_manual_creation,
            "nb_manual_import": nb_manual_import,
            "nb_first_import": nb_first_import,
            "nb_local_copies": nb_local_copies,
            "nb_moderated": nb_moderated,
        }

    def get_count_modifications(when_list):
        return [Event.get_count_modification(w) for w in when_list]
