diff --git a/catalog.yaml b/catalog.yaml index 33f873a..b5162ba 100644 --- a/catalog.yaml +++ b/catalog.yaml @@ -26,6 +26,11 @@ bottom: desc: A customizable cross-platform graphical process/system monitor for the terminal github: ClementTsang/bottom +br: + name: Broot + desc: Get an overview of a directory, even a big one + github: Canop/broot + chezmoi: desc: Manage your dotfiles across multiple diverse machines, securely github: twpayne/chezmoi @@ -218,6 +223,11 @@ rg: desc: improved grep github: BurntSushi/ripgrep +rq: + name: Record Query + desc: A tool for doing format transformation. Supports Avro, CBOR, JSON, MessagePack, Protocol Buffers, YAML, TOML, CSV + github: dflemstr/rq + sake: desc: a command runner for local and remote hosts github: alajmo/sake @@ -338,6 +348,10 @@ z: desc: A smarter cd command. Supports all major shells, inspired by z and autojump. github: ajeetdsouza/zoxide +zf: + desc: a commandline fuzzy finder designed for filtering filepaths + github: natecraddock/zf + zq: desc: process data with Zed queries github: brimdata/zed diff --git a/cliget.py b/cliget.py index 20a3df2..0769efc 100644 --- a/cliget.py +++ b/cliget.py @@ -35,7 +35,6 @@ from semver import VersionInfo import re from subprocess import run, CalledProcessError, TimeoutExpired from fuzzywuzzy import fuzz -import requests class DotDict(dict): @@ -43,13 +42,49 @@ class DotDict(dict): return self[name] if name in self else None -Tool = namedtuple('Tool','cli,props,lver,rver') -Tool.__annotations__ = {'cli':str, 'props':DotDict, 'lver':VersionInfo,'rver':VersionInfo} +Tool = namedtuple("Tool", "cli,props,lver,rver") +Tool.__annotations__ = { + "cli": str, + "props": DotDict, + "lver": VersionInfo, + "rver": VersionInfo, +} + +import requests +from requests_cache import CachedSession +from requests.adapters import HTTPAdapter +from requests.packages.urllib3.util.retry import Retry + +retry_strategy = Retry( + total=5, + status_forcelist=[403, 429, 500, 502, 503, 504], + # method_whitelist=["HEAD", "GET", "OPTIONS"], + method_whitelist=False, + backoff_factor=4, +) +adapter = HTTPAdapter(max_retries=retry_strategy) +http = CachedSession( + "cliget/http_cache", + use_cache_dir=True, # Save files in the default user cache dir + # cache_control=True, # Use Cache-Control response headers for expiration, if available + expire_after=3600, # Otherwise expire responses after one day + allowable_codes=[ + 200, + 400, + 404, + ], # Cache 400 responses as a solemn reminder of your failures + # allowable_methods=['GET', 'POST'], # Cache whatever HTTP methods you want + # ignored_parameters=['api_key'], # Don't match this request param, and redact if from the cache + # match_headers=['Accept-Language'], # Cache a different response per language + stale_if_error=True, # In case of request errors, use stale cache data if possible +) +http.mount("https://", adapter) +http.mount("http://", adapter) def trace(*mess): if "TRACE" in os.environ: - print("TRACE", mess) + print("TRACE", *mess) def info(*mess): @@ -145,16 +180,17 @@ def dosearch(options): for cli, props in ctl.items(): trace(cli, props) rtitle = fuzz.ratio(cli, pat) - rname = fuzz.ratio(props.name, pat) if 'name' in props else 0 + rname = fuzz.ratio(props.name, pat) if "name" in props else 0 rdesc = fuzz.partial_token_set_ratio(props.desc, pat) score = rdesc + rname score = rtitle + score if rtitle > 60 else score L.append((cli, props.desc[:50], score)) L = sorted(L, key=lambda x: -x[-1]) - L = [[ "cli", "desc", "rel" ]] + L[:10] + L = [["cli", "desc", "rel"]] + L[:10] from terminaltables import SingleTable + table = SingleTable(L) - table.inner_row_border=False + table.inner_row_border = False print(table.table) @@ -172,18 +208,21 @@ def _gh_versions(repo: str) -> [VersionInfo | None]: url = f"https://api.github.com/repos/{owner}/{repo}/releases" # GH API raise 403 when too many requests are sent # TODO implement retry with threshold - response = requests.get(url) + response = http.get(url) trace(response) return [_find_semver(o.get("tag_name")) for o in response.json()] -def _gh_version(repo: str) -> [VersionInfo | None]: +def _gh_version(repo: str) -> VersionInfo: [owner, repo] = repo.split("/") url = f"https://api.github.com/repos/{owner}/{repo}/releases/latest" trace(url) - response = requests.get(url) - trace(response) - return _find_semver(response.json().get("tag_name")) + response = http.get(url) + res_body = response.json() + trace(response, type(res_body), res_body) + if not response.ok: + return VersionInfo(0) + return _find_semver(res_body.get("tag_name")) def doversions(options): @@ -229,7 +268,7 @@ def doinstall(options): else: info(f"{tool} is already up do date ({lver})") else: - warn(f'{tool} has no known install strategy') + warn(f"{tool} has no known install strategy") else: warn(f"{tool} not in catalog") @@ -239,22 +278,23 @@ def _match_arch_machine(name: str) -> bool: machine = os.uname().machine.lower() # arch # we don't consider libc - glic or musl - as musl is usually statically embed lname = name.lower() - return (lname.find(sysname) > 0 - and (lname.find(machine) > 0 - or (machine == "x86_64" and lname.find("amd64") > 0)) # x86_64 and "amd64" are synonym - ) + return lname.find(sysname) > 0 and ( + lname.find(machine) > 0 or (machine == "x86_64" and lname.find("amd64") > 0) + ) # x86_64 and "amd64" are synonym + def _get_gh_matching_release(repo): # get asset list from last release url = f"https://api.github.com/repos/{repo}/releases/latest" - r = requests.get(url) + r = http.get(url) assets = r.json()["assets"] trace(assets) # select right asset asset = DotDict(next(filter(lambda x: _match_arch_machine(x["name"]), assets))) trace(asset.name, asset.url) return asset - + + def _perform_gh_install(cli, repo, version=None): asset = _get_gh_matching_release(repo) # mkdirs @@ -266,8 +306,8 @@ def _perform_gh_install(cli, repo, version=None): trace(f"will dl {location}") # dl asset if not already there if not p.exists(location): - dlurl = requests.get(asset.url).json()["browser_download_url"] - r = requests.get(dlurl, allow_redirects=True, stream=True) + dlurl = http.get(asset.url).json()["browser_download_url"] + r = http.get(dlurl, allow_redirects=True, stream=True) with open(location, "wb") as fd: shutil.copyfileobj(r.raw, fd) trace("downloaded") @@ -281,7 +321,7 @@ def _perform_gh_install(cli, repo, version=None): # TODO look for exe : ./cli, .//cli, exe propertie # symlink # TODO remove existing symlink - os.symlink(p.join('../programs', cli, cli), p.join(home, ".local/bin", cli)) + os.symlink(p.join("../programs", cli, cli), p.join(home, ".local/bin", cli)) if __name__ == "__main__": diff --git a/requirements.txt b/requirements.txt index 6bebde2..afc940d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ requests #tabulate #termtables terminaltables +requests-cache diff --git a/validate.py b/validate.py index b670e50..b2f2dbe 100644 --- a/validate.py +++ b/validate.py @@ -10,7 +10,7 @@ ctl = ldc({'__catalog':'catalog.yaml'}) if __name__ == '__main__': - report = {} + report = [] for cli, props in ctl.items(): lver, gh, rver, asset, exe = (False,)*5 # it is False until it is True # output semver on `--version` @@ -23,5 +23,12 @@ if __name__ == '__main__': # has linux + x86_64 + tgz asset # has exe at a known place r = Result(cli, lver, gh, rver, asset, exe) + report.append(r) print(r) - + tick = '\u2713' + sad = '\U0001F61E' + report = ["cli lver gh rver asset exe".split()] + [tuple(map(lambda x: ['-', tick][x] if type(x)==bool else x,r)) for r in report] + from terminaltables import SingleTable + table = SingleTable(report) + table.inner_row_border=False + print(table.table)