Merge branch 'main' into 415_Réorganiser_models
This commit is contained in:
Jean-Marie Favreau
commit
d5618c8fab
2
.gitignore
vendored
2
.gitignore
vendored
@ -52,6 +52,7 @@ cover/
|
||||
*.mo
|
||||
*.pot
|
||||
*.log
|
||||
*.log.*
|
||||
local_settings.py
|
||||
db.sqlite3
|
||||
db.sqlite3-journal
|
||||
@ -89,6 +90,7 @@ letsencrypt/
|
||||
experimentations/cache/
|
||||
experimentations/cache-augustes.ical
|
||||
experimentations/events-augustes.json
|
||||
logs-nginx
|
||||
|
||||
# MacOS
|
||||
.DS_Store
|
||||
|
||||
@ -84,3 +84,10 @@ Si la migration de la base de données s'est bien executée, mais qu'aucun évé
|
||||
|
||||
- `docker compose down --rmi all --volumes`
|
||||
- `make build-dev`
|
||||
|
||||
### Projets similaires
|
||||
|
||||
* [hackeragenda](https://github.com/YoloSwagTeam/hackeragenda)
|
||||
* [mobilizon](https://framagit.org/kaihuri/mobilizon)
|
||||
* [koalagator](https://github.com/koalagator/koalagator)
|
||||
* [gathio](https://gath.io/)
|
||||
|
||||
18
deployment/Dockerfile-nginx
Normal file
18
deployment/Dockerfile-nginx
Normal file
@ -0,0 +1,18 @@
|
||||
FROM nginx:latest
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get install -y logrotate cron && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN rm -f /var/log/nginx/access.log /var/log/nginx/error.log
|
||||
RUN chmod 755 /var/log/nginx
|
||||
RUN chown root:nginx /var/log/nginx
|
||||
|
||||
COPY deployment/scripts/nginx/nginx.logrotate /etc/logrotate.d/nginx
|
||||
RUN chmod 644 /etc/logrotate.d/nginx
|
||||
|
||||
|
||||
COPY deployment/scripts/nginx/entrypoint.sh /entrypoint.sh
|
||||
RUN chmod +x /entrypoint.sh
|
||||
|
||||
CMD ["/entrypoint.sh"]
|
||||
@ -12,5 +12,5 @@ else
|
||||
python manage.py migrate --noinput
|
||||
python manage.py collectstatic --noinput
|
||||
python manage.py compilemessages
|
||||
gunicorn "$APP_NAME".wsgi:application --bind "$APP_HOST":"$APP_PORT" --workers 3 --log-level=info
|
||||
gunicorn "$APP_NAME".wsgi:application --bind "$APP_HOST":"$APP_PORT" --workers 5 --log-level=info
|
||||
fi
|
||||
|
||||
4
deployment/scripts/nginx/entrypoint.sh
Normal file
4
deployment/scripts/nginx/entrypoint.sh
Normal file
@ -0,0 +1,4 @@
|
||||
#!/bin/bash
|
||||
cron
|
||||
|
||||
nginx -g 'daemon off;'
|
||||
@ -8,6 +8,9 @@ http {
|
||||
gzip on;
|
||||
gzip_types text/plain text/css text/javascript;
|
||||
|
||||
log_format main 'remote_addr -remote_user [time_local] "request" '
|
||||
'statusbody_bytes_sent "http_referer" '
|
||||
'"http_user_agent" "$http_x_forwarded_for"';
|
||||
|
||||
upstream backend {
|
||||
server backend:8000;
|
||||
@ -35,9 +38,11 @@ http {
|
||||
error_page 502 /static/html/502.html;
|
||||
error_page 503 /static/html/503.html;
|
||||
|
||||
if ($http_user_agent ~* "Amazonbot|meta-externalagent|ClaudeBot|ahrefsbot|semrushbot") {
|
||||
return 444;
|
||||
}
|
||||
if ($http_user_agent ~* "Amazonbot|meta-externalagent|ClaudeBot|ahrefsbot|semrushbot") {
|
||||
return 444;
|
||||
}
|
||||
access_log /var/log/nginx/access.log main;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
14
deployment/scripts/nginx/nginx.logrotate
Normal file
14
deployment/scripts/nginx/nginx.logrotate
Normal file
@ -0,0 +1,14 @@
|
||||
/var/log/nginx/*.log {
|
||||
su root nginx
|
||||
daily
|
||||
missingok
|
||||
rotate 7
|
||||
compress
|
||||
delaycompress
|
||||
notifempty
|
||||
create 640 root nginx
|
||||
sharedscripts
|
||||
postrotate
|
||||
[ -f /run/nginx.pid ] && kill -USR1 `cat /run/nginx.pid`
|
||||
endscript
|
||||
}
|
||||
@ -69,12 +69,15 @@ services:
|
||||
command: [ "/bin/bash", "/app/deployment/scripts/wait-db.sh", "/app/deployment/scripts/celery/start-beat.sh" ]
|
||||
|
||||
nginx:
|
||||
image: nginx:latest
|
||||
container_name: "${APP_NAME}-nginx"
|
||||
build:
|
||||
context: .
|
||||
dockerfile: deployment/Dockerfile-nginx
|
||||
volumes:
|
||||
- ./deployment/scripts/nginx/nginx.conf:/etc/nginx/nginx.conf:ro
|
||||
- static_files:/usr/src/app/static
|
||||
- media_files:/usr/src/app/media
|
||||
- ./logs-nginx:/var/log/nginx
|
||||
env_file: .env.prod
|
||||
ports:
|
||||
- 6380:80
|
||||
@ -86,3 +89,4 @@ volumes:
|
||||
media_files:
|
||||
postgres_data_dir:
|
||||
redis_data:
|
||||
log_files:
|
||||
|
||||
@ -24,19 +24,19 @@ from src.agenda_culturel.import_tasks.importer import URL2Events
|
||||
|
||||
if __name__ == "__main__":
|
||||
u2e = URL2Events(SimpleDownloader(), mobilizon.CExtractor())
|
||||
url = "https://mobilizon.fr/@attac63/events?"
|
||||
url_human = "https://mobilizon.fr/@attac63/events"
|
||||
url = "https://keskonfai.fr/events/166fca9c-e758-437c-8002-9a55d822e34d"
|
||||
url_human = "https://keskonfai.fr/events/166fca9c-e758-437c-8002-9a55d822e34d"
|
||||
|
||||
try:
|
||||
events = u2e.process(
|
||||
url,
|
||||
url_human,
|
||||
cache="cache-attac63.html",
|
||||
cache="cache-single-event-mobilizon.html",
|
||||
default_values={},
|
||||
published=True,
|
||||
)
|
||||
|
||||
exportfile = "events-attac63.json"
|
||||
exportfile = "events-single-event-mobilizon.json"
|
||||
print("Saving events to file {}".format(exportfile))
|
||||
with open(exportfile, "w") as f:
|
||||
json.dump(events, f, indent=4, default=str)
|
||||
|
||||
@ -458,10 +458,24 @@ def import_events_from_url(
|
||||
from .db_importer import DBImporterEvents
|
||||
|
||||
if isinstance(urls, list):
|
||||
url = urls[0]
|
||||
is_list = True
|
||||
if len(urls) > 0 and isinstance(urls[0], list):
|
||||
url = urls[0][0][0]
|
||||
cat = urls[0][0][1]
|
||||
tags = urls[0][0][2]
|
||||
if isinstance(tags, str):
|
||||
tags = [tags]
|
||||
user_id = urls[1]
|
||||
email = urls[2]
|
||||
comments = urls[3]
|
||||
is_tuple = True
|
||||
is_list = False
|
||||
else:
|
||||
url = urls[0]
|
||||
is_list = True
|
||||
is_tuple = False
|
||||
else:
|
||||
is_list = False
|
||||
is_tuple = False
|
||||
url = urls
|
||||
|
||||
with memcache_chromium_lock(self.app.oid) as acquired:
|
||||
@ -489,7 +503,8 @@ def import_events_from_url(
|
||||
|
||||
try:
|
||||
## create loader
|
||||
u2e = URL2Events(ChromiumHeadlessDownloader(), single_event=True)
|
||||
self.chromiumDownloader.pause = True
|
||||
u2e = URL2Events(self.chromiumDownloader, single_event=True)
|
||||
# set default values
|
||||
values = {}
|
||||
if cat is not None:
|
||||
@ -528,7 +543,11 @@ def import_events_from_url(
|
||||
logger.error(e)
|
||||
close_import_task(self.request.id, False, e, importer)
|
||||
|
||||
return urls[1:] if is_list else True
|
||||
return (
|
||||
urls[1:]
|
||||
if is_list
|
||||
else [urls[0][1:], user_id, email, comments] if is_tuple else True
|
||||
)
|
||||
|
||||
# if chromium is locked, we wait 30 seconds before retrying
|
||||
raise self.retry(countdown=30)
|
||||
@ -538,15 +557,20 @@ def import_events_from_url(
|
||||
def import_events_from_urls(
|
||||
self, urls_cat_tags, user_id=None, email=None, comments=None
|
||||
):
|
||||
for ucat in urls_cat_tags:
|
||||
if ucat is not None:
|
||||
url = ucat[0]
|
||||
cat = ucat[1]
|
||||
tags = ucat[2]
|
||||
|
||||
import_events_from_url.delay(
|
||||
url, cat, tags, user_id=user_id, email=email, comments=comments
|
||||
print("ça chaine baby")
|
||||
# run tasks as a chain
|
||||
tasks = chain(
|
||||
(
|
||||
import_events_from_url.s(
|
||||
[urls_cat_tags, user_id, email, comments], force=True
|
||||
)
|
||||
if i == 0
|
||||
else import_events_from_url.s(force=True)
|
||||
)
|
||||
for i in range(len(urls_cat_tags))
|
||||
)
|
||||
tasks.delay()
|
||||
|
||||
|
||||
@app.task(base=ChromiumTask, bind=True)
|
||||
|
||||
@ -555,8 +555,8 @@ class SimpleSearchEventFilter(django_filters.FilterSet):
|
||||
Func(Lower("description"), function="unaccent"), value
|
||||
),
|
||||
),
|
||||
relevance=F("rank") + F("similarity"),
|
||||
).filter(Q(rank__gte=0.5) | Q(similarity__gte=0.3))
|
||||
relevance=0.7 * F("rank") + 0.3 * F("similarity"),
|
||||
).filter(Q(rank__gte=0.1) | Q(similarity__gte=0.3))
|
||||
|
||||
for f in [
|
||||
"title",
|
||||
|
||||
@ -537,13 +537,32 @@ class EventModerateForm(ModelForm):
|
||||
class BatchImportationForm(Form):
|
||||
required_css_class = "required"
|
||||
|
||||
json = CharField(
|
||||
label="JSON",
|
||||
data = CharField(
|
||||
label=_("Data"),
|
||||
widget=Textarea(attrs={"rows": "10"}),
|
||||
help_text=_("JSON in the format expected for the import."),
|
||||
help_text=_("Supported formats: json, html."),
|
||||
required=True,
|
||||
)
|
||||
|
||||
category = ModelChoiceField(
|
||||
label=_("Category"),
|
||||
queryset=Category.objects.all().order_by("name"),
|
||||
help_text=_("Used only if data is html."),
|
||||
initial=None,
|
||||
required=False,
|
||||
)
|
||||
tags = MultipleChoiceField(
|
||||
label=_("Tags"),
|
||||
initial=None,
|
||||
choices=[],
|
||||
help_text=_("Used only if data is html."),
|
||||
required=False,
|
||||
)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self.fields["tags"].choices = Tag.get_tag_groups(all=True)
|
||||
|
||||
|
||||
class FixDuplicates(Form):
|
||||
required_css_class = "required"
|
||||
|
||||
@ -190,6 +190,9 @@ class Extractor(ABC):
|
||||
def is_known_url(url):
|
||||
return False
|
||||
|
||||
def clean_url(url):
|
||||
return url
|
||||
|
||||
def set_header(self, url):
|
||||
self.header["url"] = url
|
||||
self.header["date"] = datetime.now()
|
||||
@ -312,6 +315,7 @@ class Extractor(ABC):
|
||||
from .generic_extractors.ical import ICALExtractor
|
||||
from .custom_extractors.associations_cf import CExtractor as AssociationsCF
|
||||
from .generic_extractors.helloasso import CExtractor as HelloAssoExtractor
|
||||
from .generic_extractors.mobilizon import CExtractor as MobilizonExtractor
|
||||
|
||||
if single_event:
|
||||
return [
|
||||
@ -320,6 +324,7 @@ class Extractor(ABC):
|
||||
AssociationsCF,
|
||||
ICALExtractor,
|
||||
HelloAssoExtractor,
|
||||
MobilizonExtractor,
|
||||
EventNotFoundExtractor,
|
||||
]
|
||||
else:
|
||||
@ -372,6 +377,3 @@ class EventNotFoundExtractor(Extractor):
|
||||
)
|
||||
|
||||
return self.get_structure()
|
||||
|
||||
def clean_url(url):
|
||||
return url
|
||||
|
||||
@ -13,10 +13,50 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
# A class dedicated to get events from Mobilizon
|
||||
class CExtractor(Extractor):
|
||||
event_params = """
|
||||
id,
|
||||
title,
|
||||
url,
|
||||
beginsOn,
|
||||
endsOn,
|
||||
options {
|
||||
showStartTime,
|
||||
showEndTime,
|
||||
timezone
|
||||
},
|
||||
attributedTo {
|
||||
avatar {
|
||||
url,
|
||||
}
|
||||
name,
|
||||
preferredUsername,
|
||||
},
|
||||
description,
|
||||
onlineAddress,
|
||||
physicalAddress {
|
||||
locality,
|
||||
description,
|
||||
region
|
||||
},
|
||||
tags {
|
||||
title,
|
||||
id,
|
||||
slug
|
||||
},
|
||||
picture {
|
||||
url
|
||||
},
|
||||
status
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.no_downloader = True
|
||||
|
||||
def is_known_url(url, include_links=True):
|
||||
u = urlparse(url)
|
||||
return u.netloc in ["keskonfai.fr", "mobilizon.fr"]
|
||||
|
||||
# Source code adapted from https://framagit.org/Marc-AntoineA/mobilizon-client-python
|
||||
def _request(self, body, data):
|
||||
headers = {}
|
||||
@ -57,49 +97,20 @@ query($preferredUsername: String!, $afterDatetime: DateTime) {
|
||||
|
||||
def _oncoming_events(self):
|
||||
def _oncoming_events_page(page):
|
||||
query = """
|
||||
query = (
|
||||
"""
|
||||
query($preferredUsername: String!, $afterDatetime: DateTime, $page: Int) {
|
||||
group(preferredUsername: $preferredUsername) {
|
||||
organizedEvents(afterDatetime: $afterDatetime, page: $page) {
|
||||
elements {
|
||||
id,
|
||||
title,
|
||||
url,
|
||||
beginsOn,
|
||||
endsOn,
|
||||
options {
|
||||
showStartTime,
|
||||
showEndTime,
|
||||
timezone
|
||||
},
|
||||
attributedTo {
|
||||
avatar {
|
||||
url,
|
||||
}
|
||||
name,
|
||||
preferredUsername,
|
||||
},
|
||||
description,
|
||||
onlineAddress,
|
||||
physicalAddress {
|
||||
locality,
|
||||
description,
|
||||
region
|
||||
},
|
||||
tags {
|
||||
title,
|
||||
id,
|
||||
slug
|
||||
},
|
||||
picture {
|
||||
url
|
||||
},
|
||||
status
|
||||
elements {"""
|
||||
+ CExtractor.event_params
|
||||
+ """
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
"""
|
||||
)
|
||||
|
||||
today = datetime.now(timezone.utc).isoformat()
|
||||
data = {
|
||||
@ -119,6 +130,68 @@ query($preferredUsername: String!, $afterDatetime: DateTime, $page: Int) {
|
||||
page += 1
|
||||
return events
|
||||
|
||||
def _get_event(self):
|
||||
query = (
|
||||
"query GetEvent($uuid: UUID!) { event(uuid: $uuid) {"
|
||||
+ CExtractor.event_params
|
||||
+ "}}"
|
||||
)
|
||||
data = {
|
||||
"uuid": self._uuid_event,
|
||||
}
|
||||
|
||||
r = self._request(query, data)
|
||||
return r["event"]
|
||||
|
||||
def add_mobilizon_event(self, e, default_values, published):
|
||||
title = e["title"]
|
||||
event_url = e["url"]
|
||||
if "picture" in e and e["picture"] is not None:
|
||||
image = e["picture"]["url"]
|
||||
else:
|
||||
image = None
|
||||
location = (
|
||||
e["physicalAddress"]["description"]
|
||||
+ ", "
|
||||
+ e["physicalAddress"]["locality"]
|
||||
)
|
||||
soup = BeautifulSoup(e["description"], "html.parser")
|
||||
|
||||
description = soup.get_text(separator="\n")
|
||||
start = (
|
||||
dateutil.parser.isoparse(e["beginsOn"])
|
||||
.replace(tzinfo=timezone.utc)
|
||||
.astimezone(tz=None)
|
||||
)
|
||||
end = (
|
||||
dateutil.parser.isoparse(e["endsOn"])
|
||||
.replace(tzinfo=timezone.utc)
|
||||
.astimezone(tz=None)
|
||||
)
|
||||
|
||||
start_day = start.date()
|
||||
start_time = start.time() if e["options"]["showStartTime"] else None
|
||||
end_day = end.date()
|
||||
end_time = end.time() if e["options"]["showEndTime"] else None
|
||||
|
||||
self.add_event(
|
||||
default_values,
|
||||
title,
|
||||
None,
|
||||
start_day,
|
||||
location,
|
||||
description,
|
||||
[],
|
||||
uuids=[event_url],
|
||||
recurrences=None,
|
||||
url_human=event_url,
|
||||
start_time=start_time,
|
||||
published=published,
|
||||
image=image,
|
||||
end_day=end_day,
|
||||
end_time=end_time,
|
||||
)
|
||||
|
||||
def extract(
|
||||
self,
|
||||
content,
|
||||
@ -145,52 +218,11 @@ query($preferredUsername: String!, $afterDatetime: DateTime, $page: Int) {
|
||||
events = self._oncoming_events()
|
||||
|
||||
for e in events:
|
||||
title = e["title"]
|
||||
event_url = e["url"]
|
||||
if "picture" in e and e["picture"] is not None:
|
||||
image = e["picture"]["url"]
|
||||
else:
|
||||
image = None
|
||||
location = (
|
||||
e["physicalAddress"]["description"]
|
||||
+ ", "
|
||||
+ e["physicalAddress"]["locality"]
|
||||
)
|
||||
soup = BeautifulSoup(e["description"], "html.parser")
|
||||
|
||||
description = soup.get_text(separator="\n")
|
||||
start = (
|
||||
dateutil.parser.isoparse(e["beginsOn"])
|
||||
.replace(tzinfo=timezone.utc)
|
||||
.astimezone(tz=None)
|
||||
)
|
||||
end = (
|
||||
dateutil.parser.isoparse(e["endsOn"])
|
||||
.replace(tzinfo=timezone.utc)
|
||||
.astimezone(tz=None)
|
||||
)
|
||||
|
||||
start_day = start.date()
|
||||
start_time = start.time() if e["options"]["showStartTime"] else None
|
||||
end_day = end.date()
|
||||
end_time = end.time() if e["options"]["showEndTime"] else None
|
||||
|
||||
self.add_event(
|
||||
default_values,
|
||||
title,
|
||||
None,
|
||||
start_day,
|
||||
location,
|
||||
description,
|
||||
[],
|
||||
uuids=[event_url],
|
||||
recurrences=None,
|
||||
url_human=event_url,
|
||||
start_time=start_time,
|
||||
published=published,
|
||||
image=image,
|
||||
end_day=end_day,
|
||||
end_time=end_time,
|
||||
)
|
||||
self.add_mobilizon_event(e, default_values, published)
|
||||
elif "events" in url:
|
||||
self._api_end_point = "https://" + urlparse(url).netloc + "/api"
|
||||
self._uuid_event = url.split("/")[-1]
|
||||
event = self._get_event()
|
||||
self.add_mobilizon_event(event, default_values, published)
|
||||
|
||||
return self.get_structure()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -178,12 +178,7 @@ class DuplicatedEvents(models.Model):
|
||||
|
||||
msgs = []
|
||||
for e in self.get_duplicated():
|
||||
for m in e.message_set.filter(
|
||||
message_type__in=[
|
||||
Message.TYPE.IMPORT_PROCESS,
|
||||
Message.TYPE.UPDATE_PROCESS,
|
||||
]
|
||||
).order_by("date"):
|
||||
for m in e.message_set.order_by("date"):
|
||||
msgs.append(m)
|
||||
return msgs
|
||||
|
||||
@ -404,14 +399,7 @@ class Event(models.Model):
|
||||
self._messages = []
|
||||
|
||||
def get_import_messages(self):
|
||||
from . import Message
|
||||
|
||||
return self.message_set.filter(
|
||||
message_type__in=[
|
||||
Message.TYPE.IMPORT_PROCESS,
|
||||
Message.TYPE.UPDATE_PROCESS,
|
||||
]
|
||||
).order_by("date")
|
||||
return self.message_set.order_by("date")
|
||||
|
||||
def get_consolidated_end_day(self, intuitive=True):
|
||||
if intuitive:
|
||||
@ -593,8 +581,30 @@ class Event(models.Model):
|
||||
output_field=models.IntegerField(),
|
||||
)
|
||||
)
|
||||
with_emoji = [
|
||||
s for s in self.tags if any(char in emoji.EMOJI_DATA for char in s)
|
||||
]
|
||||
if len(with_emoji) > 0:
|
||||
qs = qs.annotate(
|
||||
overlap_emoji_tags=RawSQL(
|
||||
sql="ARRAY(select UNNEST(%s::text[]) INTERSECT select UNNEST(tags))",
|
||||
params=(with_emoji,),
|
||||
output_field=ArrayField(models.CharField(max_length=50)),
|
||||
)
|
||||
).annotate(
|
||||
overlap_emoji_tags_count=Func(
|
||||
F("overlap_emoji_tags"),
|
||||
function="CARDINALITY",
|
||||
output_field=models.IntegerField(),
|
||||
)
|
||||
)
|
||||
else:
|
||||
qs = qs.annotate(overlap_emoji_tags_count=Value(0))
|
||||
|
||||
else:
|
||||
qs = qs.annotate(overlap_tags_count=Value(1))
|
||||
qs = qs.annotate(
|
||||
overlap_tags_count=Value(0), overlap_emoji_tags_count=Value(0)
|
||||
)
|
||||
|
||||
if self.exact_location:
|
||||
qs = (
|
||||
@ -622,7 +632,8 @@ class Event(models.Model):
|
||||
)
|
||||
)
|
||||
.annotate(
|
||||
score=F("overlap_tags_count") * 30
|
||||
score=F("overlap_tags_count") * 20
|
||||
+ F("overlap_emoji_tags_count") * 40
|
||||
+ F("similarity_title") * 2
|
||||
+ F("similarity_description") * 10
|
||||
+ 10 / (F("distance") + 1)
|
||||
|
||||
@ -297,9 +297,11 @@ LOGGING = {
|
||||
"disable_existing_loggers": False,
|
||||
"handlers": {
|
||||
"file": {
|
||||
"class": "logging.handlers.RotatingFileHandler",
|
||||
"level": level_debug,
|
||||
"class": "logging.FileHandler",
|
||||
"filename": "backend.log",
|
||||
"maxBytes": 5 * 1024 * 1024, # 5 MB
|
||||
"backupCount": 5, # keep last 5 files
|
||||
},
|
||||
"mail_admins": {
|
||||
"level": "ERROR",
|
||||
|
||||
@ -521,8 +521,9 @@ header .title {
|
||||
}
|
||||
|
||||
|
||||
.slider-button {
|
||||
.slider-button a, .slider-button-inside {
|
||||
@extend [role="button"];
|
||||
color: var(--primary-inverse);
|
||||
height: 1.8em;
|
||||
width: 1.8em;
|
||||
padding: 0;
|
||||
@ -531,9 +532,6 @@ header .title {
|
||||
border-radius: .9em;
|
||||
line-height: 1.8em;
|
||||
text-align: center;
|
||||
a {
|
||||
color: var(--primary-inverse);
|
||||
}
|
||||
}
|
||||
.slider-button.hidden {
|
||||
display: none;
|
||||
@ -679,10 +677,12 @@ header .remarque {
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.highlight {
|
||||
rect.ch-subdomain-bg.highlight {
|
||||
color: var(--primary);
|
||||
font-weight: bold;
|
||||
font-style: italic;
|
||||
stroke-width: 2;
|
||||
stroke: red !important;
|
||||
}
|
||||
|
||||
.search .description {
|
||||
|
||||
@ -4,12 +4,65 @@
|
||||
{% block og_title %}Importation manuelle{% endblock %}
|
||||
{% endblock %}
|
||||
{% block content %}
|
||||
<h1>Importation manuelle</h1>
|
||||
<article>
|
||||
<form method="post">
|
||||
{% csrf_token %}
|
||||
{{ form.as_p }}
|
||||
<input type="submit" value="Envoyer">
|
||||
</form>
|
||||
</article>
|
||||
<div class="grid two-columns">
|
||||
<article>
|
||||
<header>
|
||||
<h1>Importation manuelle</h1>
|
||||
</header>
|
||||
<p>
|
||||
Le formulaire ci-dessous permet d'importer des événements par lot. Deux formats sont actuellement disponibles :
|
||||
</p>
|
||||
<ul>
|
||||
<li>
|
||||
<strong>Format json :</strong> format structuré tel qu'attendu par l'importateur. Voir <a href="https://forge.chapril.org/jmtrivial/agenda_culturel/wiki/Import-JSON">la documentation sur le wiki du projet</a>.
|
||||
</li>
|
||||
<li>
|
||||
<strong>Format html :</strong> forme libre, où tous les liens qui sont considérés comme reconnus par l'<a href="{% url 'add_event' %}">outil d'import principal</a> seront importés.
|
||||
Cette fonctionnalité peut être utile quand on veut importer plus que les 8 événements détectés par défaut lors d'un import Facebook. Pour cela, se rendre sur la page de la liste
|
||||
d'événements, enregistrer le html de la page (clic droit, enregistrer sous), puis copier le contenu de ce fichier html ci-dessous.
|
||||
</li>
|
||||
</ul>
|
||||
<p>
|
||||
Les champs catégorie et étiquettes seront utilisés si on utilise le format html, pour catégoriser tous les événements qui seront importés.
|
||||
</p>
|
||||
<form method="post">
|
||||
{% csrf_token %}
|
||||
{{ form.as_p }}
|
||||
<input type="submit" value="Envoyer">
|
||||
</form>
|
||||
</article>
|
||||
{% include "agenda_culturel/side-nav.html" with current="manual-import" %}
|
||||
</div>
|
||||
<script src="{% static 'choicejs/choices.min.js' %}"></script>
|
||||
<script>
|
||||
show_firstgroup = {
|
||||
choice(classes, choice) {
|
||||
const i = Choices.defaults.templates.choice.call(this, classes, choice);
|
||||
if (this.first_group !== null && choice.groupId == this.first_group)
|
||||
i.classList.add("visible");
|
||||
return i;
|
||||
},
|
||||
choiceGroup(classes, group) {
|
||||
const g = Choices.defaults.templates.choiceGroup.call(this, classes, group);
|
||||
if (this.first_group === undefined && group.value == "Suggestions")
|
||||
this.first_group = group.id;
|
||||
if (this.first_group !== null && group.id == this.first_group)
|
||||
g.classList.add("visible");
|
||||
return g;
|
||||
}
|
||||
};
|
||||
const tags = document.querySelector('#id_tags');
|
||||
const choices_tags = new Choices(tags,
|
||||
{
|
||||
placeholderValue: 'Sélectionner les étiquettes à ajouter',
|
||||
allowHTML: true,
|
||||
searchResultLimit: 10,
|
||||
delimiter: ',',
|
||||
removeItemButton: true,
|
||||
shouldSort: false,
|
||||
callbackOnCreateTemplates: () => (show_firstgroup)
|
||||
}
|
||||
);
|
||||
|
||||
</script>
|
||||
{% endblock %}
|
||||
|
||||
@ -76,7 +76,7 @@
|
||||
choices.showDropdown();
|
||||
|
||||
setTimeout(() => {
|
||||
const searchTerm = htmlDecode('{{ object.location }}');
|
||||
const searchTerm = htmlDecode('{{ object.location }}').replace(/\(?\d{5}\)?/, '');
|
||||
choices.input.focus();
|
||||
choices.input.value = searchTerm;
|
||||
choices._handleSearch(searchTerm);
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
{% block content %}
|
||||
<article>
|
||||
<header>
|
||||
<h1>Importer un événement</h1>
|
||||
<h1>Importer un événement</h1>
|
||||
{% url 'event_import_url' as local_url %}
|
||||
{% include "agenda_culturel/static_content.html" with name="import" url_path=local_url %}
|
||||
</header>
|
||||
@ -53,6 +53,7 @@
|
||||
placeholderValue: 'Sélectionner les étiquettes à ajouter',
|
||||
allowHTML: true,
|
||||
delimiter: ',',
|
||||
searchResultLimit: 10,
|
||||
removeItemButton: true,
|
||||
shouldSort: false,
|
||||
callbackOnCreateTemplates: () => (show_firstgroup)
|
||||
|
||||
@ -60,6 +60,7 @@
|
||||
{
|
||||
placeholderValue: 'Sélectionner les étiquettes à ajouter',
|
||||
allowHTML: true,
|
||||
searchResultLimit: 10,
|
||||
delimiter: ',',
|
||||
removeItemButton: true,
|
||||
shouldSort: false,
|
||||
|
||||
@ -70,6 +70,10 @@
|
||||
<a {% if current == "imports" %}class="selected"{% endif %}
|
||||
href="{% url 'imports' %}">Historiques des importations</a>
|
||||
</li>
|
||||
<li>
|
||||
<a {% if current == "manual-import" %}class="selected"{% endif %}
|
||||
href="{% url 'add_import' %}">Import manuel</a>
|
||||
</li>
|
||||
{% endif %}
|
||||
{% if perms.agenda_culturel.view_recurrentimport %}
|
||||
<li>
|
||||
|
||||
@ -132,7 +132,14 @@
|
||||
},
|
||||
theme: (window.matchMedia && window.matchMedia('(prefers-color-scheme: dark)').matches) ? "dark" : "light",
|
||||
itemSelector: '#cal-heatmap-startday',
|
||||
animationDuration: 0
|
||||
animationDuration: 0,
|
||||
scale: {
|
||||
color: {
|
||||
type: 'linear',
|
||||
domain: [0, Math.max(...data_startday.map(d => d.value))],
|
||||
range: ['#f0fefe', '#005e5e'],
|
||||
},
|
||||
}
|
||||
};
|
||||
|
||||
// create first heatmap
|
||||
@ -146,6 +153,12 @@
|
||||
options.itemSelector = '#cal-heatmap-creation';
|
||||
options.date.start = new Date("{{ first_by_creation.isoformat }}");
|
||||
options.data.source = data_creation;
|
||||
options.scale.color = {
|
||||
type: 'linear',
|
||||
domain: [0, Math.max(...data_creation.map(d => d.value))],
|
||||
range: ['#fdf5f5', '#5e0000'],
|
||||
};
|
||||
|
||||
const cal_creation = new CalHeatmap();
|
||||
cal_creation.paint(options, [
|
||||
[CalendarLabel, calendarlabel],
|
||||
|
||||
@ -340,7 +340,7 @@ def import_event_proxy(request):
|
||||
)
|
||||
return HttpResponseRedirect(ex.get_absolute_url())
|
||||
else:
|
||||
messages.info(
|
||||
messages.success(
|
||||
request,
|
||||
_(
|
||||
"This type of address is known to the calendar, so an automatic import is proposed."
|
||||
|
||||
@ -10,9 +10,12 @@ from django.urls import reverse_lazy
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from ..celery import app as celery_app, update_orphan_pure_import_events
|
||||
from ..celery import import_events_from_json
|
||||
from ..celery import import_events_from_json, import_events_from_urls
|
||||
from ..forms import BatchImportationForm
|
||||
from ..models import Event, BatchImportation, RecurrentImport
|
||||
from ..import_tasks.extractor import Extractor
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
|
||||
|
||||
@login_required(login_url="/accounts/login/")
|
||||
@ -72,9 +75,38 @@ def add_import(request):
|
||||
form = BatchImportationForm(request.POST)
|
||||
|
||||
if form.is_valid():
|
||||
import_events_from_json.delay(form.data["json"])
|
||||
try:
|
||||
# try to load data as a json file
|
||||
json.loads(form.data["data"])
|
||||
# if data is a json, load it
|
||||
import_events_from_json.delay(form.data["data"])
|
||||
messages.success(
|
||||
request, _("The import from json has been run successfully.")
|
||||
)
|
||||
except ValueError:
|
||||
# otherwise, consider it as html, extract all urls, and import them
|
||||
soup = BeautifulSoup(form.data["data"], "html.parser")
|
||||
urls = list(
|
||||
set(
|
||||
[
|
||||
a["href"]
|
||||
for a in soup.find_all("a", href=True)
|
||||
if Extractor.is_known_url_default_extractors(a["href"])
|
||||
]
|
||||
)
|
||||
)
|
||||
# then import events from url
|
||||
import_events_from_urls.delay(
|
||||
[(u, form.data["category"], form.data["tags"]) for u in urls],
|
||||
user_id=request.user.pk if request.user else None,
|
||||
)
|
||||
messages.success(
|
||||
request,
|
||||
_(
|
||||
"The import from html ({} detected links) has been run successfully."
|
||||
).format(len(urls)),
|
||||
)
|
||||
|
||||
messages.success(request, _("The import has been run successfully."))
|
||||
return HttpResponseRedirect(reverse_lazy("imports"))
|
||||
|
||||
return render(request, "agenda_culturel/batchimportation_form.html", {"form": form})
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user