chore: package

src/instances.py

@@ -0,0 +1,93 @@
+import sys
+import os
+import logging
+import time
+import sqlite3
+from scrapy import Spider, Request
+from scrapy.http import Response
+from model import Model
+
+logger = logging.getLogger(__name__)
+
+
+def yield_from_db():
+    with sqlite3.connect(f'file:{os.environ.get("DATADIR")}/sqlpage.db?mode=ro', uri=True) as db:
+        for row in db.execute('select rowid, domain from instances where failure < 5'):
+            yield row
+
+def yield_tests():
+    #yield (105, 'keskonfai.fr',)
+    yield (7, 'mobilizon.fr',)
+
+class Fedator(Spider):
+    name = "fedator"
+    custom_settings = {
+        "USER_AGENT": "Mozilla/5.0 (X11; Linux x86_64; rv:100.0) Gecko/20100101 Firefox/100.0",
+        "ROBOTSTXT_OBEY": False,
+        "REFERER_ENABLED": False,
+        "COOKIES_ENABLED": False,
+        "TELNETCONSOLE_ENABLED": False,
+        "HTTPCACHE_ENABLED": False,
+        "DEFAULT_REQUEST_HEADERS": {
+            "Accept": "application/json",
+            "Content-Type": "application/json",
+        },
+        "DOWNLOAD_TIMEOUT": 10,
+    }
+
+
+    async def start(self):
+        gql_query = open('src/query.gql', 'r').read()
+        from json import dumps
+        body = dumps({"query": gql_query})
+        # bbody = bytes(body, encoding='utf-8')
+        # sys.stdout.buffer.write(bbody)
+        for row in yield_tests():
+            domain = row[1]
+            yield Request(
+                url = f"https://{domain}/api",
+                method = 'POST',
+                body = body,
+                meta = { "record": row, },
+            )
+
+    def parse(self, response: Response):
+        res:Model = Model.model_validate_json(response.body)
+        meta = response.request.meta
+        instance_id = meta.get('record')[0]
+        with sqlite3.connect(f'file:{os.environ.get("DATADIR")}/sqlpage.db', timeout=15.0, isolation_level='IMMEDIATE') as db:
+            # insert new stats
+            stats = res.data.statistics
+            db.execute('''insert into stats(insertedAt, instance_id, users, local_groups, total_groups, local_events, total_events, local_comments, total_comments, following, followers) values (?,?,?,?,?,?,?,?,?,?,?)
+                ''', (
+                   int(time.time())  # insertedAt
+                 , instance_id
+                 , stats.number_of_users # users
+                 , stats.number_of_local_groups
+                 , stats.number_of_groups
+                 , stats.number_of_local_events
+                 , stats.number_of_events
+                 , stats.number_of_local_comments
+                 , stats.number_of_comments
+                 , stats.number_of_instance_followings
+                 , stats.number_of_instance_followers
+                    
+                ))
+            # update info
+            config = res.data.config
+            db.execute('''update instances set name=?,slogan=?,description=?,languages=?,open=?,version=?
+                -- ,location=?
+                ,failure=?,updatedAt=? where rowid=?                 
+                ''', (
+                      config.name
+                    , config.slogan
+                    , config.description
+                    , ",".join(config.languages)
+                    , config.registrations_open
+                    , config.version
+                    #, config.location
+                    , 0  # failure
+                    , int(time.time())  # updatedAt
+                    , instance_id
+                ))
+        

src/model.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import Any, List, Optional
+
+from pydantic import BaseModel, Field
+
+
+class Config(BaseModel):
+    country_code: Any = Field(..., alias='countryCode')
+    description: str
+    languages: List[str]
+    long_description: str = Field(..., alias='longDescription')
+    name: str
+    registrations_open: bool = Field(..., alias='registrationsOpen')
+    slogan: str
+    version: str
+
+
+class Statistics(BaseModel):
+    number_of_comments: int = Field(..., alias='numberOfComments')
+    number_of_events: int = Field(..., alias='numberOfEvents')
+    number_of_groups: int = Field(..., alias='numberOfGroups')
+    number_of_instance_followers: int = Field(..., alias='numberOfInstanceFollowers')
+    number_of_instance_followings: int = Field(..., alias='numberOfInstanceFollowings')
+    number_of_local_comments: int = Field(..., alias='numberOfLocalComments')
+    number_of_local_events: int = Field(..., alias='numberOfLocalEvents')
+    number_of_local_groups: int = Field(..., alias='numberOfLocalGroups')
+    number_of_users: int = Field(..., alias='numberOfUsers')
+
+
+class Data(BaseModel):
+    config: Config
+    statistics: Statistics
+
+
+class Model(BaseModel):
+    data: Data
+

src/query.gql

@@ -0,0 +1,23 @@
+query About {
+  statistics {
+    numberOfUsers
+    numberOfLocalGroups
+    numberOfGroups
+    numberOfLocalEvents
+    numberOfEvents
+    numberOfLocalComments
+    numberOfComments
+    numberOfInstanceFollowings
+    numberOfInstanceFollowers
+  }
+  config {
+    name
+    version
+    registrationsOpen
+    slogan
+    description
+    longDescription
+    countryCode
+    languages
+  }
+}