synit
/
pmbootstrap
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Make APKINDEX downloads returning 404 non-fatal (!1726) 

Do not fail when an APKINDEX can not be downloaded, print a WARNING
instead. This matches, what apk does. Add a new allow_404 parameter to
pmb.helpers.http.download(), and use it in pmb.helpers.repo.update()
for downloading APKINDEX files. Cache the APKINDEX URLs that gave a 404
for the session, so we do not attempt to download these again.

This is needed for the new binary repository: the initial build is done
without existing APKINDEX files, so we must not fail in that case.
This commit is contained in:
Oliver Smith 2018-12-11 07:44:32 +01:00
parent fa7860c8f6
commit 5102d48064
3 changed files with 28 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
pmb/helpers/args.py
View File

@ -130,7 +130,7 @@ def add_shortcuts(args):
def add_cache(args):
""" Add a caching dict (caches parsing of files etc. for the current
session) """
repo_update = {"offline_msg_shown": False}
repo_update = {"404": [], "offline_msg_shown": False}
setattr(args, "cache", {"apkindex": {},
"apkbuild": {},
"apk_min_version_checked": [],

21
pmb/helpers/http.py
View File

@ -24,7 +24,8 @@ import urllib.request
import pmb.helpers.run
def download(args, url, prefix, cache=True, loglevel=logging.INFO):
def download(args, url, prefix, cache=True, loglevel=logging.INFO,
allow_404=False):
""" Download a file to disk.
:param url: the http(s) address of to the file to download
@ -34,6 +35,9 @@ def download(args, url, prefix, cache=True, loglevel=logging.INFO):
message in 'pmbootstrap log', not in stdout. We use
this when downloading many APKINDEX files at once, no
point in showing a dozen messages.
:param allow_404: do not raise an exception when the server responds
with a 404 Not Found error. Only display a warning on
stdout (no matter if loglevel is changed).
:returns: path to the downloaded file in the cache or None on 404 """
# Create cache folder
if not os.path.exists(args.work + "/cache_http"):
@ -50,7 +54,16 @@ def download(args, url, prefix, cache=True, loglevel=logging.INFO):
# Download the file
logging.log(loglevel, "Download " + url)
with urllib.request.urlopen(url) as response:
with open(path, "wb") as handle:
shutil.copyfileobj(response, handle)
try:
with urllib.request.urlopen(url) as response:
with open(path, "wb") as handle:
shutil.copyfileobj(response, handle)
# Handle 404
except urllib.error.HTTPError as e:
if e.code == 404 and allow_404:
logging.warning("WARNING: file not found: " + url)
return None
raise
# Return path in cache
return path

13
pmb/helpers/repo.py
View File

@ -144,9 +144,13 @@ def update(args, arch=None, force=False, existing_only=False):
cache_apk_outside = args.work + "/cache_apk_" + arch
apkindex = cache_apk_outside + "/APKINDEX." + hash(url) + ".tar.gz"
# Find update reason, possibly skip non-existing files
# Find update reason, possibly skip non-existing or known 404 files
reason = None
if not os.path.exists(apkindex):
if url_full in args.cache[cache_key]["404"]:
# We already attempted to download this file once in this
# session
continue
elif not os.path.exists(apkindex):
if existing_only:
continue
reason = "file does not exist yet"
@ -172,7 +176,10 @@ def update(args, arch=None, force=False, existing_only=False):
# Download and move to right location
for url, target in outdated.items():
temp = pmb.helpers.http.download(args, url, "APKINDEX", False,
logging.DEBUG)
logging.DEBUG, True)
if not temp:
args.cache[cache_key]["404"].append(url)
continue
target_folder = os.path.dirname(target)
if not os.path.exists(target_folder):
pmb.helpers.run.root(args, ["mkdir", "-p", target_folder])