diff --git a/download_manager.py b/download_manager.py new file mode 100644 index 0000000..eed281b --- /dev/null +++ b/download_manager.py @@ -0,0 +1,42 @@ +from typing import Awaitable, Dict, Callable, Optional, Tuple + +from http3 import AsyncClient +from yaml import safe_load + + +async def init_download_manager(): + await DownloadManager.load_remote_resources({ + "linguist": ("https://cdn.jsdelivr.net/gh/github/linguist@master/lib/linguist/languages.yml", {}) + }) + + +class DownloadManager: + _client = AsyncClient() + _REMOTE_RESOURCES = dict() + + @staticmethod + async def load_remote_resources(resources: Dict[str, Tuple[str, Dict]]): + for resource, (url, params) in resources.items(): + DownloadManager._REMOTE_RESOURCES[resource] = DownloadManager._client.get(url, **params) + + @staticmethod + async def _get_remote_resource(resource: str, convertor: Optional[Callable[[bytes], str]]) -> Dict: + if isinstance(DownloadManager._REMOTE_RESOURCES[resource], Awaitable): + res = await DownloadManager._REMOTE_RESOURCES[resource] + if res.status_code == 200: + if convertor is None: + DownloadManager._REMOTE_RESOURCES[resource] = res.json() + print(res.json()) + else: + DownloadManager._REMOTE_RESOURCES[resource] = convertor(res.content) + else: + raise Exception(f"Query '{res.url}' failed to run by returning code of {res.status_code}: {res.json()}") + return DownloadManager._REMOTE_RESOURCES[resource] + + @staticmethod + async def get_remote_json(resource: str) -> Dict: + return await DownloadManager._get_remote_resource(resource, None) + + @staticmethod + async def get_remote_yaml(resource: str) -> Dict: + return await DownloadManager._get_remote_resource(resource, safe_load) diff --git a/main.py b/main.py index f8c1bc9..45f82ec 100644 --- a/main.py +++ b/main.py @@ -4,12 +4,16 @@ Readme Development Metrics With waka time progress import re import os import base64 +from asyncio import run + from pytz import timezone import pytz import requests from github import Github, GithubException, InputGitAuthor import datetime from string import Template + +from download_manager import init_download_manager from loc import LinesOfCode import time import traceback @@ -543,6 +547,7 @@ def generate_new_readme(stats: str, readme: str): if __name__ == '__main__': + run(init_download_manager()) try: start_time = datetime.datetime.now().timestamp() * 1000 if ghtoken is None: diff --git a/make_bar_graph.py b/make_bar_graph.py index 94f06cd..22307fd 100644 --- a/make_bar_graph.py +++ b/make_bar_graph.py @@ -1,9 +1,15 @@ import os +from asyncio import run + import pandas as pd import numpy as np import altair as alt import json import os + +from download_manager import DownloadManager + + # npm install vega-lite vega-cli canvas @@ -14,8 +20,7 @@ class BarGraph: def build_graph(self): - with open(os.path.join(os.path.dirname(__file__), 'colors.json')) as f: - colors = json.load(f) + colors = run(DownloadManager.get_remote_yaml("linguist")) allColorsValues = [] # filter data diff --git a/requirements.txt b/requirements.txt index cdc1bf0..089c05b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,7 @@ chardet==4.0.0 cycler==0.10.0 Deprecated==1.2.12 entrypoints==0.3 +http3==0.6.7 humanize==3.3.0 idna==2.10 Jinja2==2.11.3 @@ -21,6 +22,7 @@ Pillow==8.2.0 portpicker==1.3.1 PyGithub==1.54.1 PyJWT==1.7.1 +PyYAML==6.0 pyparsing==2.4.7 pyrsistent==0.17.3 python-dateutil==2.8.1