Make APKINDEX downloads returning 404 non-fatal (!1726)

Do not fail when an APKINDEX can not be downloaded, print a WARNING
instead. This matches, what apk does. Add a new allow_404 parameter to
pmb.helpers.http.download(), and use it in pmb.helpers.repo.update()
for downloading APKINDEX files. Cache the APKINDEX URLs that gave a 404
for the session, so we do not attempt to download these again.

This is needed for the new binary repository: the initial build is done
without existing APKINDEX files, so we must not fail in that case.

pmb/helpers/args.py

@@ -130,7 +130,7 @@ def add_shortcuts(args):
 def add_cache(args):
     """ Add a caching dict (caches parsing of files etc. for the current
         session) """
-    repo_update = {"offline_msg_shown": False}
+    repo_update = {"404": [], "offline_msg_shown": False}
     setattr(args, "cache", {"apkindex": {},
                             "apkbuild": {},
                             "apk_min_version_checked": [],

pmb/helpers/http.py

@@ -24,7 +24,8 @@ import urllib.request
 import pmb.helpers.run
 
 
-def download(args, url, prefix, cache=True, loglevel=logging.INFO):
+def download(args, url, prefix, cache=True, loglevel=logging.INFO,
+             allow_404=False):
     """ Download a file to disk.
 
         :param url: the http(s) address of to the file to download
@@ -34,6 +35,9 @@ def download(args, url, prefix, cache=True, loglevel=logging.INFO):
                          message in 'pmbootstrap log', not in stdout. We use
                          this when downloading many APKINDEX files at once, no
                          point in showing a dozen messages.
+        :param allow_404: do not raise an exception when the server responds
+                          with a 404 Not Found error. Only display a warning on
+                          stdout (no matter if loglevel is changed).
         :returns: path to the downloaded file in the cache or None on 404 """
     # Create cache folder
     if not os.path.exists(args.work + "/cache_http"):
@@ -50,7 +54,16 @@ def download(args, url, prefix, cache=True, loglevel=logging.INFO):
 
     # Download the file
     logging.log(loglevel, "Download " + url)
-    with urllib.request.urlopen(url) as response:
+    try:
-        with open(path, "wb") as handle:
+        with urllib.request.urlopen(url) as response:
-            shutil.copyfileobj(response, handle)
+            with open(path, "wb") as handle:
+                shutil.copyfileobj(response, handle)
+    # Handle 404
+    except urllib.error.HTTPError as e:
+        if e.code == 404 and allow_404:
+            logging.warning("WARNING: file not found: " + url)
+            return None
+        raise
+
+    # Return path in cache
     return path

pmb/helpers/repo.py

@@ -144,9 +144,13 @@ def update(args, arch=None, force=False, existing_only=False):
             cache_apk_outside = args.work + "/cache_apk_" + arch
             apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz"
 
-            # Find update reason, possibly skip non-existing files
+            # Find update reason, possibly skip non-existing or known 404 files
             reason = None
-            if not os.path.exists(apkindex):
+            if url_full in args.cache[cache_key]["404"]:
+                # We already attempted to download this file once in this
+                # session
+                continue
+            elif not os.path.exists(apkindex):
                 if existing_only:
                     continue
                 reason = "file does not exist yet"
@@ -172,7 +176,10 @@ def update(args, arch=None, force=False, existing_only=False):
     # Download and move to right location
     for url, target in outdated.items():
         temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
-                                         logging.DEBUG)
+                                         logging.DEBUG, True)
+        if not temp:
+            args.cache[cache_key]["404"].append(url)
+            continue
         target_folder = os.path.dirname(target)
         if not os.path.exists(target_folder):
             pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])