almost working impl

catalog.yaml

@@ -26,6 +26,11 @@ bottom:
   desc: A customizable cross-platform graphical process/system monitor for the terminal
   github: ClementTsang/bottom
 
+br:
+  name: Broot
+  desc: Get an overview of a directory, even a big one
+  github: Canop/broot
+
 chezmoi:
   desc: Manage your dotfiles across multiple diverse machines, securely
   github: twpayne/chezmoi
@@ -218,6 +223,11 @@ rg:
   desc: improved grep
   github: BurntSushi/ripgrep
 
+rq:
+  name: Record Query
+  desc: A tool for doing format transformation. Supports Avro, CBOR, JSON, MessagePack, Protocol Buffers, YAML, TOML, CSV
+  github: dflemstr/rq
+
 sake:
   desc: a command runner for local and remote hosts
   github: alajmo/sake
@@ -338,6 +348,10 @@ z:
   desc: A smarter cd command. Supports all major shells, inspired by z and autojump.
   github: ajeetdsouza/zoxide
 
+zf:
+  desc: a commandline fuzzy finder designed for filtering filepaths
+  github: natecraddock/zf
+
 zq:
   desc: process data with Zed queries
   github: brimdata/zed

cliget.py

@@ -35,7 +35,6 @@ from semver import VersionInfo
 import re
 from subprocess import run, CalledProcessError, TimeoutExpired
 from fuzzywuzzy import fuzz
-import requests
 
 
 class DotDict(dict):
@@ -43,13 +42,49 @@ class DotDict(dict):
         return self[name] if name in self else None
 
 
-Tool = namedtuple('Tool','cli,props,lver,rver')
+Tool = namedtuple("Tool", "cli,props,lver,rver")
-Tool.__annotations__ = {'cli':str, 'props':DotDict, 'lver':VersionInfo,'rver':VersionInfo}
+Tool.__annotations__ = {
+    "cli": str,
+    "props": DotDict,
+    "lver": VersionInfo,
+    "rver": VersionInfo,
+}
+
+import requests
+from requests_cache import CachedSession
+from requests.adapters import HTTPAdapter
+from requests.packages.urllib3.util.retry import Retry
+
+retry_strategy = Retry(
+    total=5,
+    status_forcelist=[403, 429, 500, 502, 503, 504],
+    # method_whitelist=["HEAD", "GET", "OPTIONS"],
+    method_whitelist=False,
+    backoff_factor=4,
+)
+adapter = HTTPAdapter(max_retries=retry_strategy)
+http = CachedSession(
+    "cliget/http_cache",
+    use_cache_dir=True,  # Save files in the default user cache dir
+    # cache_control=True,                # Use Cache-Control response headers for expiration, if available
+    expire_after=3600,  # Otherwise expire responses after one day
+    allowable_codes=[
+        200,
+        400,
+        404,
+    ],  # Cache 400 responses as a solemn reminder of your failures
+    # allowable_methods=['GET', 'POST'], # Cache whatever HTTP methods you want
+    # ignored_parameters=['api_key'],    # Don't match this request param, and redact if from the cache
+    # match_headers=['Accept-Language'], # Cache a different response per language
+    stale_if_error=True,  # In case of request errors, use stale cache data if possible
+)
+http.mount("https://", adapter)
+http.mount("http://", adapter)
 
 
 def trace(*mess):
     if "TRACE" in os.environ:
-        print("TRACE", mess)
+        print("TRACE", *mess)
 
 
 def info(*mess):
@@ -145,7 +180,7 @@ def dosearch(options):
     for cli, props in ctl.items():
         trace(cli, props)
         rtitle = fuzz.ratio(cli, pat)
-        rname = fuzz.ratio(props.name, pat) if 'name' in props else 0
+        rname = fuzz.ratio(props.name, pat) if "name" in props else 0
         rdesc = fuzz.partial_token_set_ratio(props.desc, pat)
         score = rdesc + rname
         score = rtitle + score if rtitle > 60 else score
@@ -153,6 +188,7 @@ def dosearch(options):
     L = sorted(L, key=lambda x: -x[-1])
     L = [["cli", "desc", "rel"]] + L[:10]
     from terminaltables import SingleTable
+
     table = SingleTable(L)
     table.inner_row_border = False
     print(table.table)
@@ -172,18 +208,21 @@ def _gh_versions(repo: str) -> [VersionInfo | None]:
     url = f"https://api.github.com/repos/{owner}/{repo}/releases"
     # GH API raise 403 when too many requests are sent
     # TODO implement retry with threshold
-    response = requests.get(url)
+    response = http.get(url)
     trace(response)
     return [_find_semver(o.get("tag_name")) for o in response.json()]
 
 
-def _gh_version(repo: str) -> [VersionInfo | None]:
+def _gh_version(repo: str) -> VersionInfo:
     [owner, repo] = repo.split("/")
     url = f"https://api.github.com/repos/{owner}/{repo}/releases/latest"
     trace(url)
-    response = requests.get(url)
+    response = http.get(url)
-    trace(response)
+    res_body = response.json()
-    return _find_semver(response.json().get("tag_name"))
+    trace(response, type(res_body), res_body)
+    if not response.ok:
+        return VersionInfo(0)
+    return _find_semver(res_body.get("tag_name"))
 
 
 def doversions(options):
@@ -229,7 +268,7 @@ def doinstall(options):
                 else:
                     info(f"{tool} is already up do date ({lver})")
             else:
-                warn(f'{tool} has no known install strategy')
+                warn(f"{tool} has no known install strategy")
         else:
             warn(f"{tool} not in catalog")
 
@@ -239,15 +278,15 @@ def _match_arch_machine(name: str) -> bool:
     machine = os.uname().machine.lower()  # arch
     # we don't consider libc - glic or musl - as musl is usually statically embed
     lname = name.lower()
-    return (lname.find(sysname) > 0
+    return lname.find(sysname) > 0 and (
-        and (lname.find(machine) > 0
+        lname.find(machine) > 0 or (machine == "x86_64" and lname.find("amd64") > 0)
-             or (machine == "x86_64" and lname.find("amd64") > 0))  # x86_64 and "amd64" are synonym
+    )  # x86_64 and "amd64" are synonym
-    )
+
 
 def _get_gh_matching_release(repo):
     # get asset list from last release
     url = f"https://api.github.com/repos/{repo}/releases/latest"
-    r = requests.get(url)
+    r = http.get(url)
     assets = r.json()["assets"]
     trace(assets)
     # select right asset
@@ -255,6 +294,7 @@ def _get_gh_matching_release(repo):
     trace(asset.name, asset.url)
     return asset
 
+
 def _perform_gh_install(cli, repo, version=None):
     asset = _get_gh_matching_release(repo)
     # mkdirs
@@ -266,8 +306,8 @@ def _perform_gh_install(cli, repo, version=None):
     trace(f"will dl {location}")
     # dl asset if not already there
     if not p.exists(location):
-        dlurl = requests.get(asset.url).json()["browser_download_url"]
+        dlurl = http.get(asset.url).json()["browser_download_url"]
-        r = requests.get(dlurl, allow_redirects=True, stream=True)
+        r = http.get(dlurl, allow_redirects=True, stream=True)
         with open(location, "wb") as fd:
             shutil.copyfileobj(r.raw, fd)
         trace("downloaded")
@@ -281,7 +321,7 @@ def _perform_gh_install(cli, repo, version=None):
     # TODO look for exe : ./cli, ./<tar root folder>/cli, exe propertie
     # symlink
     # TODO remove existing symlink
-    os.symlink(p.join('../programs', cli, cli), p.join(home, ".local/bin", cli))
+    os.symlink(p.join("../programs", cli, cli), p.join(home, ".local/bin", cli))
 
 
 if __name__ == "__main__":

requirements.txt

@@ -8,3 +8,4 @@ requests
 #tabulate
 #termtables
 terminaltables
+requests-cache

validate.py

@@ -10,7 +10,7 @@ ctl = ldc({'__catalog':'catalog.yaml'})
 
 
 if __name__ == '__main__':
-    report = {}
+    report = []
     for cli, props in ctl.items():
         lver, gh, rver, asset, exe = (False,)*5  # it is False until it is True
         # output semver on `--version`
@@ -23,5 +23,12 @@ if __name__ == '__main__':
             # has linux + x86_64 + tgz asset
             # has exe at a known place
         r = Result(cli, lver, gh, rver, asset, exe)
+        report.append(r)
         print(r)
-   
+    tick = '\u2713'
+    sad = '\U0001F61E'
+    report = ["cli lver gh rver asset exe".split()] + [tuple(map(lambda x: ['-', tick][x] if type(x)==bool else x,r)) for r in report]
+    from terminaltables import SingleTable
+    table = SingleTable(report)
+    table.inner_row_border=False
+    print(table.table)