download manager implemented for linguist

download_manager.py

@@ -0,0 +1,42 @@
+from typing import Awaitable, Dict, Callable, Optional, Tuple
+
+from http3 import AsyncClient
+from yaml import safe_load
+
+
+async def init_download_manager():
+    await DownloadManager.load_remote_resources({
+        "linguist": ("https://cdn.jsdelivr.net/gh/github/linguist@master/lib/linguist/languages.yml", {})
+    })
+
+
+class DownloadManager:
+    _client = AsyncClient()
+    _REMOTE_RESOURCES = dict()
+
+    @staticmethod
+    async def load_remote_resources(resources: Dict[str, Tuple[str, Dict]]):
+        for resource, (url, params) in resources.items():
+            DownloadManager._REMOTE_RESOURCES[resource] = DownloadManager._client.get(url, **params)
+
+    @staticmethod
+    async def _get_remote_resource(resource: str, convertor: Optional[Callable[[bytes], str]]) -> Dict:
+        if isinstance(DownloadManager._REMOTE_RESOURCES[resource], Awaitable):
+            res = await DownloadManager._REMOTE_RESOURCES[resource]
+            if res.status_code == 200:
+                if convertor is None:
+                    DownloadManager._REMOTE_RESOURCES[resource] = res.json()
+                    print(res.json())
+                else:
+                    DownloadManager._REMOTE_RESOURCES[resource] = convertor(res.content)
+            else:
+                raise Exception(f"Query '{res.url}' failed to run by returning code of {res.status_code}: {res.json()}")
+        return DownloadManager._REMOTE_RESOURCES[resource]
+
+    @staticmethod
+    async def get_remote_json(resource: str) -> Dict:
+        return await DownloadManager._get_remote_resource(resource, None)
+
+    @staticmethod
+    async def get_remote_yaml(resource: str) -> Dict:
+        return await DownloadManager._get_remote_resource(resource, safe_load)

main.py

@@ -4,12 +4,16 @@ Readme Development Metrics With waka time progress
 import re
 import os
 import base64
+from asyncio import run
+
 from pytz import timezone
 import pytz
 import requests
 from github import Github, GithubException, InputGitAuthor
 import datetime
 from string import Template
+
+from download_manager import init_download_manager
 from loc import LinesOfCode
 import time
 import traceback
@@ -543,6 +547,7 @@ def generate_new_readme(stats: str, readme: str):
 
 
 if __name__ == '__main__':
+    run(init_download_manager())
     try:
         start_time = datetime.datetime.now().timestamp() * 1000
         if ghtoken is None:

make_bar_graph.py

@@ -1,9 +1,15 @@
 import os
+from asyncio import run
+
 import pandas as pd
 import numpy as np
 import altair as alt
 import json
 import os
+
+from download_manager import DownloadManager
+
+
 # npm install vega-lite vega-cli canvas
 
 
@@ -14,8 +20,7 @@ class BarGraph:
 
     def build_graph(self):
         
-        with open(os.path.join(os.path.dirname(__file__), 'colors.json')) as f:
-            colors = json.load(f)
+        colors = run(DownloadManager.get_remote_yaml("linguist"))
         allColorsValues = []
 
         # filter data

requirements.txt

@@ -8,6 +8,7 @@ chardet==4.0.0
 cycler==0.10.0
 Deprecated==1.2.12
 entrypoints==0.3
+http3==0.6.7
 humanize==3.3.0
 idna==2.10
 Jinja2==2.11.3
@@ -21,6 +22,7 @@ Pillow==8.2.0
 portpicker==1.3.1
 PyGithub==1.54.1
 PyJWT==1.7.1
+PyYAML==6.0
 pyparsing==2.4.7
 pyrsistent==0.17.3
 python-dateutil==2.8.1