Merge branch 'main' into 415_Réorganiser_models

.gitignore

@@ -52,6 +52,7 @@ cover/
 *.mo
 *.pot
 *.log
+*.log.*
 local_settings.py
 db.sqlite3
 db.sqlite3-journal
@@ -89,6 +90,7 @@ letsencrypt/
 experimentations/cache/
 experimentations/cache-augustes.ical
 experimentations/events-augustes.json
+logs-nginx
 
 # MacOS
 .DS_Store

README.md

@@ -84,3 +84,10 @@ Si la migration de la base de données s'est bien executée, mais qu'aucun évé
 
 - `docker compose down --rmi all --volumes`
 - `make build-dev`
+
+### Projets similaires
+
+* [hackeragenda](https://github.com/YoloSwagTeam/hackeragenda)
+* [mobilizon](https://framagit.org/kaihuri/mobilizon)
+* [koalagator](https://github.com/koalagator/koalagator)
+* [gathio](https://gath.io/)

deployment/Dockerfile-nginx

@@ -0,0 +1,18 @@
+FROM nginx:latest
+
+RUN apt-get update && \
+    apt-get install -y logrotate cron && \
+    rm -rf /var/lib/apt/lists/*
+
+RUN rm -f /var/log/nginx/access.log /var/log/nginx/error.log
+RUN chmod 755 /var/log/nginx
+RUN chown root:nginx /var/log/nginx
+
+COPY deployment/scripts/nginx/nginx.logrotate /etc/logrotate.d/nginx
+RUN chmod 644 /etc/logrotate.d/nginx
+
+
+COPY deployment/scripts/nginx/entrypoint.sh /entrypoint.sh
+RUN chmod +x /entrypoint.sh
+
+CMD ["/entrypoint.sh"]

deployment/scripts/backend/start.sh

@@ -12,5 +12,5 @@ else
     python manage.py migrate --noinput
     python manage.py collectstatic --noinput
     python manage.py compilemessages
-    gunicorn "$APP_NAME".wsgi:application --bind "$APP_HOST":"$APP_PORT" --workers 3 --log-level=info
+    gunicorn "$APP_NAME".wsgi:application --bind "$APP_HOST":"$APP_PORT" --workers 5 --log-level=info
 fi

deployment/scripts/nginx/entrypoint.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+cron
+
+nginx -g 'daemon off;'

deployment/scripts/nginx/nginx.conf

@@ -8,6 +8,9 @@ http {
     gzip on;
     gzip_types text/plain text/css text/javascript;
 
+    log_format main 'remote_addr -remote_user [time_local] "request" '
+                'statusbody_bytes_sent "http_referer" '
+                '"http_user_agent" "$http_x_forwarded_for"';
 
     upstream backend {
         server backend:8000;
@@ -38,6 +41,8 @@ http {
         if ($http_user_agent ~* "Amazonbot|meta-externalagent|ClaudeBot|ahrefsbot|semrushbot") {
             return 444;
         }
+        access_log  /var/log/nginx/access.log main;
+        error_log   /var/log/nginx/error.log warn;
 
     }
 }

deployment/scripts/nginx/nginx.logrotate

@@ -0,0 +1,14 @@
+/var/log/nginx/*.log {
+    su root nginx
+    daily
+    missingok
+    rotate 7
+    compress
+    delaycompress
+    notifempty
+    create 640 root nginx
+    sharedscripts
+    postrotate
+        [ -f /run/nginx.pid ] && kill -USR1 `cat /run/nginx.pid`
+    endscript
+}

docker-compose.prod.yml

@@ -69,12 +69,15 @@ services:
     command: [ "/bin/bash", "/app/deployment/scripts/wait-db.sh", "/app/deployment/scripts/celery/start-beat.sh" ]
 
   nginx:
-    image: nginx:latest
     container_name: "${APP_NAME}-nginx"
+    build:
+      context: .
+      dockerfile: deployment/Dockerfile-nginx
     volumes:
       - ./deployment/scripts/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
       - static_files:/usr/src/app/static
       - media_files:/usr/src/app/media
+      - ./logs-nginx:/var/log/nginx
     env_file: .env.prod
     ports:
       - 6380:80
@@ -86,3 +89,4 @@ volumes:
   media_files:
   postgres_data_dir:
   redis_data:
+  log_files:

experimentations/get_mobilizon.py

@@ -24,19 +24,19 @@ from src.agenda_culturel.import_tasks.importer import URL2Events
 
 if __name__ == "__main__":
     u2e = URL2Events(SimpleDownloader(), mobilizon.CExtractor())
-    url = "https://mobilizon.fr/@attac63/events?"
+    url = "https://keskonfai.fr/events/166fca9c-e758-437c-8002-9a55d822e34d"
-    url_human = "https://mobilizon.fr/@attac63/events"
+    url_human = "https://keskonfai.fr/events/166fca9c-e758-437c-8002-9a55d822e34d"
 
     try:
         events = u2e.process(
             url,
             url_human,
-            cache="cache-attac63.html",
+            cache="cache-single-event-mobilizon.html",
             default_values={},
             published=True,
         )
 
-        exportfile = "events-attac63.json"
+        exportfile = "events-single-event-mobilizon.json"
         print("Saving events to file {}".format(exportfile))
         with open(exportfile, "w") as f:
             json.dump(events, f, indent=4, default=str)

src/agenda_culturel/celery.py

@@ -458,10 +458,24 @@ def import_events_from_url(
     from .db_importer import DBImporterEvents
 
     if isinstance(urls, list):
+        if len(urls) > 0 and isinstance(urls[0], list):
+            url = urls[0][0][0]
+            cat = urls[0][0][1]
+            tags = urls[0][0][2]
+            if isinstance(tags, str):
+                tags = [tags]
+            user_id = urls[1]
+            email = urls[2]
+            comments = urls[3]
+            is_tuple = True
+            is_list = False
+        else:
             url = urls[0]
             is_list = True
+            is_tuple = False
     else:
         is_list = False
+        is_tuple = False
         url = urls
 
     with memcache_chromium_lock(self.app.oid) as acquired:
@@ -489,7 +503,8 @@ def import_events_from_url(
 
                 try:
                     ## create loader
-                    u2e = URL2Events(ChromiumHeadlessDownloader(), single_event=True)
+                    self.chromiumDownloader.pause = True
+                    u2e = URL2Events(self.chromiumDownloader, single_event=True)
                     # set default values
                     values = {}
                     if cat is not None:
@@ -528,7 +543,11 @@ def import_events_from_url(
                     logger.error(e)
                     close_import_task(self.request.id, False, e, importer)
 
-            return urls[1:] if is_list else True
+            return (
+                urls[1:]
+                if is_list
+                else [urls[0][1:], user_id, email, comments] if is_tuple else True
+            )
 
     # if chromium is locked, we wait 30 seconds before retrying
     raise self.retry(countdown=30)
@@ -538,15 +557,20 @@ def import_events_from_url(
 def import_events_from_urls(
     self, urls_cat_tags, user_id=None, email=None, comments=None
 ):
-    for ucat in urls_cat_tags:
-        if ucat is not None:
-            url = ucat[0]
-            cat = ucat[1]
-            tags = ucat[2]
 
-            import_events_from_url.delay(
+    print("ça chaine baby")
-                url, cat, tags, user_id=user_id, email=email, comments=comments
+    # run tasks as a chain
+    tasks = chain(
+        (
+            import_events_from_url.s(
+                [urls_cat_tags, user_id, email, comments], force=True
             )
+            if i == 0
+            else import_events_from_url.s(force=True)
+        )
+        for i in range(len(urls_cat_tags))
+    )
+    tasks.delay()
 
 
 @app.task(base=ChromiumTask, bind=True)

src/agenda_culturel/filters.py

@@ -555,8 +555,8 @@ class SimpleSearchEventFilter(django_filters.FilterSet):
                     Func(Lower("description"), function="unaccent"), value
                 ),
             ),
-            relevance=F("rank") + F("similarity"),
+            relevance=0.7 * F("rank") + 0.3 * F("similarity"),
-        ).filter(Q(rank__gte=0.5) | Q(similarity__gte=0.3))
+        ).filter(Q(rank__gte=0.1) | Q(similarity__gte=0.3))
 
         for f in [
             "title",

src/agenda_culturel/forms.py

@@ -537,13 +537,32 @@ class EventModerateForm(ModelForm):
 class BatchImportationForm(Form):
     required_css_class = "required"
 
-    json = CharField(
+    data = CharField(
-        label="JSON",
+        label=_("Data"),
         widget=Textarea(attrs={"rows": "10"}),
-        help_text=_("JSON in the format expected for the import."),
+        help_text=_("Supported formats: json, html."),
         required=True,
     )
 
+    category = ModelChoiceField(
+        label=_("Category"),
+        queryset=Category.objects.all().order_by("name"),
+        help_text=_("Used only if data is html."),
+        initial=None,
+        required=False,
+    )
+    tags = MultipleChoiceField(
+        label=_("Tags"),
+        initial=None,
+        choices=[],
+        help_text=_("Used only if data is html."),
+        required=False,
+    )
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.fields["tags"].choices = Tag.get_tag_groups(all=True)
+
 
 class FixDuplicates(Form):
     required_css_class = "required"

src/agenda_culturel/import_tasks/extractor.py

@@ -190,6 +190,9 @@ class Extractor(ABC):
     def is_known_url(url):
         return False
 
+    def clean_url(url):
+        return url
+
     def set_header(self, url):
         self.header["url"] = url
         self.header["date"] = datetime.now()
@@ -312,6 +315,7 @@ class Extractor(ABC):
         from .generic_extractors.ical import ICALExtractor
         from .custom_extractors.associations_cf import CExtractor as AssociationsCF
         from .generic_extractors.helloasso import CExtractor as HelloAssoExtractor
+        from .generic_extractors.mobilizon import CExtractor as MobilizonExtractor
 
         if single_event:
             return [
@@ -320,6 +324,7 @@ class Extractor(ABC):
                 AssociationsCF,
                 ICALExtractor,
                 HelloAssoExtractor,
+                MobilizonExtractor,
                 EventNotFoundExtractor,
             ]
         else:
@@ -372,6 +377,3 @@ class EventNotFoundExtractor(Extractor):
         )
 
         return self.get_structure()
-
-    def clean_url(url):
-        return url

src/agenda_culturel/import_tasks/generic_extractors/mobilizon.py

@@ -13,10 +13,50 @@ logger = logging.getLogger(__name__)
 
 # A class dedicated to get events from Mobilizon
 class CExtractor(Extractor):
+    event_params = """
+            id,
+            title,
+            url,
+            beginsOn,
+            endsOn,
+            options {
+                showStartTime,
+                showEndTime,
+                timezone
+            },
+            attributedTo {
+                avatar {
+                    url,
+                }
+                name,
+                preferredUsername,
+            },
+            description,
+            onlineAddress,
+            physicalAddress {
+                locality,
+                description,
+                region
+            },
+            tags {
+                title,
+                id,
+                slug
+            },
+            picture {
+                url
+            },
+            status
+    """
+
     def __init__(self):
         super().__init__()
         self.no_downloader = True
 
+    def is_known_url(url, include_links=True):
+        u = urlparse(url)
+        return u.netloc in ["keskonfai.fr", "mobilizon.fr"]
+
     # Source code adapted from https://framagit.org/Marc-AntoineA/mobilizon-client-python
     def _request(self, body, data):
         headers = {}
@@ -57,49 +97,20 @@ query($preferredUsername: String!, $afterDatetime: DateTime) {
 
     def _oncoming_events(self):
         def _oncoming_events_page(page):
-            query = """
+            query = (
+                """
 query($preferredUsername: String!, $afterDatetime: DateTime, $page: Int) {
     group(preferredUsername: $preferredUsername) {
         organizedEvents(afterDatetime: $afterDatetime, page: $page) {
-            elements {
+            elements {"""
-                id,
+                + CExtractor.event_params
-                title,
+                + """
-                url,
-                beginsOn,
-                endsOn,
-                options {
-                showStartTime,
-                showEndTime,
-                timezone
-                },
-                attributedTo {
-                avatar {
-                    url,
-                }
-                name,
-                preferredUsername,
-                },
-                description,
-                onlineAddress,
-                physicalAddress {
-                locality,
-                description,
-                region
-                },
-                tags {
-                title,
-                id,
-                slug
-                },
-                picture {
-                url
-                },
-                status
             }
         }
     }
 }
         """
+            )
 
             today = datetime.now(timezone.utc).isoformat()
             data = {
@@ -119,32 +130,20 @@ query($preferredUsername: String!, $afterDatetime: DateTime, $page: Int) {
             page += 1
         return events
 
-    def extract(
+    def _get_event(self):
-        self,
+        query = (
-        content,
+            "query GetEvent($uuid: UUID!) { event(uuid: $uuid) {"
-        url,
+            + CExtractor.event_params
-        url_human=None,
+            + "}}"
-        default_values=None,
+        )
-        published=False,
+        data = {
-    ):
+            "uuid": self._uuid_event,
-        self.set_header(url)
+        }
-        self.clear_events()
 
-        if "@" in url:
+        r = self._request(query, data)
-            # split url to identify server url and actor id
+        return r["event"]
-            elems = [x for x in url.split("/") if len(x) > 0 and x[0] == "@"]
-            if len(elems) == 1:
-                params = elems[0].split("@")
-                if len(params) == 2:
-                    self._api_end_point = "https://" + urlparse(url).netloc + "/api"
-                    self._group_id = params[1]
-                else:
-                    self._api_end_point = "https://" + params[2] + "/api"
-                    self._group_id = params[1]
 
-                events = self._oncoming_events()
+    def add_mobilizon_event(self, e, default_values, published):
-
-                for e in events:
         title = e["title"]
         event_url = e["url"]
         if "picture" in e and e["picture"] is not None:
@@ -193,4 +192,37 @@ query($preferredUsername: String!, $afterDatetime: DateTime, $page: Int) {
             end_time=end_time,
         )
 
+    def extract(
+        self,
+        content,
+        url,
+        url_human=None,
+        default_values=None,
+        published=False,
+    ):
+        self.set_header(url)
+        self.clear_events()
+
+        if "@" in url:
+            # split url to identify server url and actor id
+            elems = [x for x in url.split("/") if len(x) > 0 and x[0] == "@"]
+            if len(elems) == 1:
+                params = elems[0].split("@")
+                if len(params) == 2:
+                    self._api_end_point = "https://" + urlparse(url).netloc + "/api"
+                    self._group_id = params[1]
+                else:
+                    self._api_end_point = "https://" + params[2] + "/api"
+                    self._group_id = params[1]
+
+                events = self._oncoming_events()
+
+                for e in events:
+                    self.add_mobilizon_event(e, default_values, published)
+        elif "events" in url:
+            self._api_end_point = "https://" + urlparse(url).netloc + "/api"
+            self._uuid_event = url.split("/")[-1]
+            event = self._get_event()
+            self.add_mobilizon_event(event, default_values, published)
+
         return self.get_structure()

src/agenda_culturel/models/event.py

@@ -178,12 +178,7 @@ class DuplicatedEvents(models.Model):
 
         msgs = []
         for e in self.get_duplicated():
-            for m in e.message_set.filter(
+            for m in e.message_set.order_by("date"):
-                message_type__in=[
-                    Message.TYPE.IMPORT_PROCESS,
-                    Message.TYPE.UPDATE_PROCESS,
-                ]
-            ).order_by("date"):
                 msgs.append(m)
         return msgs
 
@@ -404,14 +399,7 @@ class Event(models.Model):
         self._messages = []
 
     def get_import_messages(self):
-        from . import Message
+        return self.message_set.order_by("date")
-
-        return self.message_set.filter(
-            message_type__in=[
-                Message.TYPE.IMPORT_PROCESS,
-                Message.TYPE.UPDATE_PROCESS,
-            ]
-        ).order_by("date")
 
     def get_consolidated_end_day(self, intuitive=True):
         if intuitive:
@@ -593,8 +581,30 @@ class Event(models.Model):
                         output_field=models.IntegerField(),
                     )
                 )
+                with_emoji = [
+                    s for s in self.tags if any(char in emoji.EMOJI_DATA for char in s)
+                ]
+                if len(with_emoji) > 0:
+                    qs = qs.annotate(
+                        overlap_emoji_tags=RawSQL(
+                            sql="ARRAY(select UNNEST(%s::text[]) INTERSECT select UNNEST(tags))",
+                            params=(with_emoji,),
+                            output_field=ArrayField(models.CharField(max_length=50)),
+                        )
+                    ).annotate(
+                        overlap_emoji_tags_count=Func(
+                            F("overlap_emoji_tags"),
+                            function="CARDINALITY",
+                            output_field=models.IntegerField(),
+                        )
+                    )
                 else:
-                qs = qs.annotate(overlap_tags_count=Value(1))
+                    qs = qs.annotate(overlap_emoji_tags_count=Value(0))
+
+            else:
+                qs = qs.annotate(
+                    overlap_tags_count=Value(0), overlap_emoji_tags_count=Value(0)
+                )
 
             if self.exact_location:
                 qs = (
@@ -622,7 +632,8 @@ class Event(models.Model):
                     )
                 )
                 .annotate(
-                    score=F("overlap_tags_count") * 30
+                    score=F("overlap_tags_count") * 20
+                    + F("overlap_emoji_tags_count") * 40
                     + F("similarity_title") * 2
                     + F("similarity_description") * 10
                     + 10 / (F("distance") + 1)

src/agenda_culturel/settings/base.py

@@ -297,9 +297,11 @@ LOGGING = {
     "disable_existing_loggers": False,
     "handlers": {
         "file": {
+            "class": "logging.handlers.RotatingFileHandler",
             "level": level_debug,
-            "class": "logging.FileHandler",
             "filename": "backend.log",
+            "maxBytes": 5 * 1024 * 1024,  # 5 MB
+            "backupCount": 5,  # keep last 5 files
         },
         "mail_admins": {
             "level": "ERROR",

src/agenda_culturel/static/style.scss

@@ -521,8 +521,9 @@ header .title {
     }
 
 
-    .slider-button {
+    .slider-button a, .slider-button-inside {
         @extend [role="button"];
+        color: var(--primary-inverse);
         height: 1.8em;
         width: 1.8em;
         padding: 0;
@@ -531,9 +532,6 @@ header .title {
         border-radius: .9em;
         line-height: 1.8em;
         text-align: center;
-        a {
-            color: var(--primary-inverse);
-        }
     }
     .slider-button.hidden {
         display: none;
@@ -679,10 +677,12 @@ header .remarque {
     display: inline-block;
 }
 
-.highlight {
+rect.ch-subdomain-bg.highlight {
     color: var(--primary);
     font-weight: bold;
     font-style: italic;
+    stroke-width: 2;
+    stroke: red !important;
 }
 
 .search .description {

src/agenda_culturel/templates/agenda_culturel/batchimportation_form.html

@@ -4,12 +4,65 @@
   {% block og_title %}Importation manuelle{% endblock %}
 {% endblock %}
 {% block content %}
-  <h1>Importation manuelle</h1>
+  <div class="grid two-columns">
     <article>
+      <header>
+        <h1>Importation manuelle</h1>
+      </header>
+      <p>
+        Le formulaire ci-dessous permet d'importer des événements par lot. Deux formats sont actuellement disponibles&nbsp;:
+      </p>
+      <ul>
+        <li>
+          <strong>Format json&nbsp;:</strong> format structuré tel qu'attendu par l'importateur. Voir <a href="https://forge.chapril.org/jmtrivial/agenda_culturel/wiki/Import-JSON">la documentation sur le wiki du projet</a>.
+        </li>
+        <li>
+          <strong>Format html&nbsp;:</strong> forme libre, où tous les liens qui sont considérés comme reconnus par l'<a href="{% url 'add_event' %}">outil d'import principal</a> seront importés.
+          Cette fonctionnalité peut être utile quand on veut importer plus que les 8 événements détectés par défaut lors d'un import Facebook. Pour cela, se rendre sur la page de la liste
+          d'événements, enregistrer le html de la page (clic droit, enregistrer sous), puis copier le contenu de ce fichier html ci-dessous.
+        </li>
+      </ul>
+      <p>
+        Les champs catégorie et étiquettes seront utilisés si on utilise le format html, pour catégoriser tous les événements qui seront importés.
+      </p>
       <form method="post">
         {% csrf_token %}
         {{ form.as_p }}
         <input type="submit" value="Envoyer">
       </form>
     </article>
+    {% include "agenda_culturel/side-nav.html" with current="manual-import" %}
+  </div>
+  <script src="{% static 'choicejs/choices.min.js' %}"></script>
+  <script>
+    show_firstgroup = {
+                choice(classes, choice) {
+                    const i = Choices.defaults.templates.choice.call(this, classes, choice);
+                    if (this.first_group !== null && choice.groupId == this.first_group)
+                        i.classList.add("visible");
+                    return i;
+                },
+                choiceGroup(classes, group) {
+                    const g = Choices.defaults.templates.choiceGroup.call(this, classes, group);
+                    if (this.first_group === undefined && group.value == "Suggestions")
+                        this.first_group = group.id;
+                    if (this.first_group !== null && group.id == this.first_group)
+                       g.classList.add("visible");
+                    return g;
+                }
+    };
+    const tags = document.querySelector('#id_tags');
+    const choices_tags = new Choices(tags,
+        {
+            placeholderValue: 'Sélectionner les étiquettes à ajouter',
+            allowHTML: true,
+            searchResultLimit: 10,
+            delimiter: ',',
+            removeItemButton: true,
+            shouldSort: false,
+            callbackOnCreateTemplates: () => (show_firstgroup)
+        }
+    );
+
+  </script>
 {% endblock %}

src/agenda_culturel/templates/agenda_culturel/event_set_place_form.html

@@ -76,7 +76,7 @@
     choices.showDropdown();
 
     setTimeout(() => {
-      const searchTerm = htmlDecode('{{ object.location }}');
+      const searchTerm = htmlDecode('{{ object.location }}').replace(/\(?\d{5}\)?/, '');
       choices.input.focus();
       choices.input.value = searchTerm;
       choices._handleSearch(searchTerm);

src/agenda_culturel/templates/agenda_culturel/import.html

@@ -53,6 +53,7 @@
             placeholderValue: 'Sélectionner les étiquettes à ajouter',
             allowHTML: true,
             delimiter: ',',
+            searchResultLimit: 10,
             removeItemButton: true,
             shouldSort: false,
             callbackOnCreateTemplates: () => (show_firstgroup)

src/agenda_culturel/templates/agenda_culturel/import_set.html

@@ -60,6 +60,7 @@
             {
                 placeholderValue: 'Sélectionner les étiquettes à ajouter',
                 allowHTML: true,
+                searchResultLimit: 10,
                 delimiter: ',',
                 removeItemButton: true,
                 shouldSort: false,

src/agenda_culturel/templates/agenda_culturel/side-nav.html

@@ -70,6 +70,10 @@
               <a {% if current == "imports" %}class="selected"{% endif %}
                  href="{% url 'imports' %}">Historiques des importations</a>
             </li>
+            <li>
+              <a {% if current == "manual-import" %}class="selected"{% endif %}
+                 href="{% url 'add_import' %}">Import manuel</a>
+            </li>
           {% endif %}
           {% if perms.agenda_culturel.view_recurrentimport %}
             <li>

src/agenda_culturel/templates/agenda_culturel/statistics.html

@@ -132,7 +132,14 @@
         },
         theme: (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? "dark" : "light",
         itemSelector: '#cal-heatmap-startday',
-        animationDuration: 0
+        animationDuration: 0,
+        scale: {
+          color: {
+            type: 'linear',
+            domain: [0, Math.max(...data_startday.map(d => d.value))],
+            range: ['#f0fefe', '#005e5e'],
+          },
+        }
     };
 
     // create first heatmap
@@ -146,6 +153,12 @@
     options.itemSelector = '#cal-heatmap-creation';
     options.date.start =  new Date("{{ first_by_creation.isoformat }}");
     options.data.source = data_creation;
+    options.scale.color = {
+        type: 'linear',
+        domain: [0, Math.max(...data_creation.map(d => d.value))],
+        range: ['#fdf5f5', '#5e0000'],
+      };
+
     const cal_creation = new CalHeatmap();
     cal_creation.paint(options, [
         [CalendarLabel, calendarlabel],

src/agenda_culturel/views/event_views.py

@@ -340,7 +340,7 @@ def import_event_proxy(request):
                     )
                     return HttpResponseRedirect(ex.get_absolute_url())
                 else:
-                    messages.info(
+                    messages.success(
                         request,
                         _(
                             "This type of address is known to the calendar, so an automatic import is proposed."

src/agenda_culturel/views/import_batch_views.py

@@ -10,9 +10,12 @@ from django.urls import reverse_lazy
 from django.utils.translation import gettext_lazy as _
 
 from ..celery import app as celery_app, update_orphan_pure_import_events
-from ..celery import import_events_from_json
+from ..celery import import_events_from_json, import_events_from_urls
 from ..forms import BatchImportationForm
 from ..models import Event, BatchImportation, RecurrentImport
+from ..import_tasks.extractor import Extractor
+from bs4 import BeautifulSoup
+import json
 
 
 @login_required(login_url="/accounts/login/")
@@ -72,9 +75,38 @@ def add_import(request):
         form = BatchImportationForm(request.POST)
 
         if form.is_valid():
-            import_events_from_json.delay(form.data["json"])
+            try:
+                # try to load data as a json file
+                json.loads(form.data["data"])
+                # if data is a json, load it
+                import_events_from_json.delay(form.data["data"])
+                messages.success(
+                    request, _("The import from json has been run successfully.")
+                )
+            except ValueError:
+                # otherwise, consider it as html, extract all urls, and import them
+                soup = BeautifulSoup(form.data["data"], "html.parser")
+                urls = list(
+                    set(
+                        [
+                            a["href"]
+                            for a in soup.find_all("a", href=True)
+                            if Extractor.is_known_url_default_extractors(a["href"])
+                        ]
+                    )
+                )
+                # then import events from url
+                import_events_from_urls.delay(
+                    [(u, form.data["category"], form.data["tags"]) for u in urls],
+                    user_id=request.user.pk if request.user else None,
+                )
+                messages.success(
+                    request,
+                    _(
+                        "The import from html ({} detected links) has been run successfully."
+                    ).format(len(urls)),
+                )
 
-            messages.success(request, _("The import has been run successfully."))
             return HttpResponseRedirect(reverse_lazy("imports"))
 
     return render(request, "agenda_culturel/batchimportation_form.html", {"form": form})