diff --git a/pmb/helpers/args.py b/pmb/helpers/args.py index 29a0b720..a7cbd6e4 100644 --- a/pmb/helpers/args.py +++ b/pmb/helpers/args.py @@ -130,7 +130,7 @@ def add_shortcuts(args): def add_cache(args): """ Add a caching dict (caches parsing of files etc. for the current session) """ - repo_update = {"offline_msg_shown": False} + repo_update = {"404": [], "offline_msg_shown": False} setattr(args, "cache", {"apkindex": {}, "apkbuild": {}, "apk_min_version_checked": [], diff --git a/pmb/helpers/http.py b/pmb/helpers/http.py index 6212dc0e..8aabad5e 100644 --- a/pmb/helpers/http.py +++ b/pmb/helpers/http.py @@ -24,7 +24,8 @@ import urllib.request import pmb.helpers.run -def download(args, url, prefix, cache=True, loglevel=logging.INFO): +def download(args, url, prefix, cache=True, loglevel=logging.INFO, + allow_404=False): """ Download a file to disk. :param url: the http(s) address of to the file to download @@ -34,6 +35,9 @@ def download(args, url, prefix, cache=True, loglevel=logging.INFO): message in 'pmbootstrap log', not in stdout. We use this when downloading many APKINDEX files at once, no point in showing a dozen messages. + :param allow_404: do not raise an exception when the server responds + with a 404 Not Found error. Only display a warning on + stdout (no matter if loglevel is changed). :returns: path to the downloaded file in the cache or None on 404 """ # Create cache folder if not os.path.exists(args.work + "/cache_http"): @@ -50,7 +54,16 @@ def download(args, url, prefix, cache=True, loglevel=logging.INFO): # Download the file logging.log(loglevel, "Download " + url) - with urllib.request.urlopen(url) as response: - with open(path, "wb") as handle: - shutil.copyfileobj(response, handle) + try: + with urllib.request.urlopen(url) as response: + with open(path, "wb") as handle: + shutil.copyfileobj(response, handle) + # Handle 404 + except urllib.error.HTTPError as e: + if e.code == 404 and allow_404: + logging.warning("WARNING: file not found: " + url) + return None + raise + + # Return path in cache return path diff --git a/pmb/helpers/repo.py b/pmb/helpers/repo.py index 861a6d6c..96ba6783 100644 --- a/pmb/helpers/repo.py +++ b/pmb/helpers/repo.py @@ -144,9 +144,13 @@ def update(args, arch=None, force=False, existing_only=False): cache_apk_outside = args.work + "/cache_apk_" + arch apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz" - # Find update reason, possibly skip non-existing files + # Find update reason, possibly skip non-existing or known 404 files reason = None - if not os.path.exists(apkindex): + if url_full in args.cache[cache_key]["404"]: + # We already attempted to download this file once in this + # session + continue + elif not os.path.exists(apkindex): if existing_only: continue reason = "file does not exist yet" @@ -172,7 +176,10 @@ def update(args, arch=None, force=False, existing_only=False): # Download and move to right location for url, target in outdated.items(): temp = pmb.helpers.http.download(args, url, "APKINDEX", False, - logging.DEBUG) + logging.DEBUG, True) + if not temp: + args.cache[cache_key]["404"].append(url) + continue target_folder = os.path.dirname(target) if not os.path.exists(target_folder): pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])