APKBUILD parser: recognize all top-level variables (MR 2300)
A common pattern in APKBUILDs, is to introduce custom variables prefixed with underscores that get then used in makedepends and other variables. For example: _wlrootsmakedepends=" eudev-dev # ... " makedepends=" # ... $_wlrootsmakedepends " Adjust the APKBUILD parser code, so it parses all top-level variables and can use them further below when referenced inside other variables. Before returning the parsed APKBUILD data, remove all variables that are not in pmbootstrap's list of known APKBUILD parsing attributes (so the result is the same). I've compared "pmbootstrap apkbuild_parse" (which parses all APKBUILDs in the currently checked out pmaports dir), before and after this change, and the result is the same except for having more variables successfully replaced. - Performance Note- This new implementation is actually faster than the previous one, because we don't need to iterate through all known keys on each line of the APKBUILDs. On my machine, average of 3 runs, parsing all APKBUILDs of current pmaports master takes about half as long as with the previous implementation. $ time pmbootstrap -q apkbuild_parse >/dev/null -> old code: 0.954 -> new code: 0.483
This commit is contained in:
Oliver Smith
parent
8efee86388
commit
85ee201cf5
|
|
@ -24,6 +24,9 @@ revar3 = re.compile(r"\${([a-zA-Z_]+[a-zA-Z0-9_]*)/([^/]+)(?:/([^/]*?))?}")
|
|||
# ${foo#bar} -- cut off bar from foo from start of string
|
||||
revar4 = re.compile(r"\${([a-zA-Z_]+[a-zA-Z0-9_]*)#(.*)}")
|
||||
|
||||
# foo=
|
||||
revar5 = re.compile(r"([a-zA-Z_]+[a-zA-Z0-9_]*)=")
|
||||
|
||||
|
||||
def replace_variable(apkbuild, value: str) -> str:
|
||||
def log_key_not_found(match):
|
||||
|
|
@ -122,7 +125,7 @@ def read_file(path):
|
|||
return lines
|
||||
|
||||
|
||||
def parse_attribute(attribute, lines, i, path):
|
||||
def parse_next_attribute(lines, i, path):
|
||||
"""
|
||||
Parse one attribute from the APKBUILD.
|
||||
|
||||
|
|
@ -136,19 +139,21 @@ def parse_attribute(attribute, lines, i, path):
|
|||
first-pkg
|
||||
second-pkg"
|
||||
|
||||
:param attribute: from the APKBUILD, i.e. "pkgname"
|
||||
:param lines: \n-terminated list of lines from the APKBUILD
|
||||
:param i: index of the line we are currently looking at
|
||||
:param path: full path to the APKBUILD (for error message)
|
||||
:returns: (found, value, i)
|
||||
found: True if the attribute was found in line i, False otherwise
|
||||
:returns: (attribute, value, i)
|
||||
attribute: attribute name if any was found in line i / None
|
||||
value: that was parsed from the line
|
||||
i: line that was parsed last
|
||||
"""
|
||||
# Check for and cut off "attribute="
|
||||
if not lines[i].startswith(attribute + "="):
|
||||
return (False, None, i)
|
||||
value = lines[i][len(attribute + "="):-1]
|
||||
rematch5 = revar5.match(lines[i])
|
||||
if not rematch5:
|
||||
return (None, None, i)
|
||||
attribute = rematch5.group(0)
|
||||
value = lines[i][len(attribute):-1]
|
||||
attribute = rematch5.group(0).rstrip("=")
|
||||
|
||||
# Determine end quote sign
|
||||
end_char = None
|
||||
|
|
@ -161,10 +166,10 @@ def parse_attribute(attribute, lines, i, path):
|
|||
# Single line
|
||||
if not end_char:
|
||||
value = value.split("#")[0].rstrip()
|
||||
return (True, value, i)
|
||||
return (attribute, value, i)
|
||||
if end_char in value:
|
||||
value = value.split(end_char, 1)[0]
|
||||
return (True, value, i)
|
||||
return (attribute, value, i)
|
||||
|
||||
# Parse lines until reaching end quote
|
||||
i += 1
|
||||
|
|
@ -173,7 +178,7 @@ def parse_attribute(attribute, lines, i, path):
|
|||
value += " "
|
||||
if end_char in line:
|
||||
value += line.split(end_char, 1)[0].strip()
|
||||
return (True, value.strip(), i)
|
||||
return (attribute, value.strip(), i)
|
||||
value += line.strip()
|
||||
i += 1
|
||||
|
||||
|
|
@ -191,13 +196,12 @@ def _parse_attributes(path, lines, apkbuild_attributes, ret):
|
|||
:param apkbuild_attributes: the attributes to parse
|
||||
:param ret: a dict to update with new parsed variable
|
||||
"""
|
||||
# Parse all variables first, and replace variables mentioned earlier
|
||||
for i in range(len(lines)):
|
||||
for attribute, options in apkbuild_attributes.items():
|
||||
found, value, i = parse_attribute(attribute, lines, i, path)
|
||||
if not found:
|
||||
continue
|
||||
|
||||
ret[attribute] = replace_variable(ret, value)
|
||||
attribute, value, i = parse_next_attribute(lines, i, path)
|
||||
if not attribute:
|
||||
continue
|
||||
ret[attribute] = replace_variable(ret, value)
|
||||
|
||||
if "subpackages" in apkbuild_attributes:
|
||||
subpackages = OrderedDict()
|
||||
|
|
@ -217,6 +221,11 @@ def _parse_attributes(path, lines, apkbuild_attributes, ret):
|
|||
else:
|
||||
ret[attribute] = 0
|
||||
|
||||
# Remove variables not in attributes
|
||||
for attribute in list(ret.keys()):
|
||||
if attribute not in apkbuild_attributes:
|
||||
del ret[attribute]
|
||||
|
||||
|
||||
def _parse_subpackage(path, lines, apkbuild, subpackages, subpkg):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -82,51 +82,53 @@ def test_depends_in_depends(args):
|
|||
assert apkbuild["depends"] == ["first", "second", "third"]
|
||||
|
||||
|
||||
def test_parse_attributes(args):
|
||||
def test_parse_next_attribute(args):
|
||||
# Convenience function for calling the function with a block of text
|
||||
def func(attribute, block):
|
||||
def func(block):
|
||||
lines = block.split("\n")
|
||||
for i in range(0, len(lines)):
|
||||
lines[i] += "\n"
|
||||
i = 0
|
||||
path = "(testcase in " + __file__ + ")"
|
||||
print("=== parsing attribute '" + attribute + "' in test block:")
|
||||
print("=== parsing next attribute in test block:")
|
||||
print(block)
|
||||
print("===")
|
||||
return pmb.parse._apkbuild.parse_attribute(attribute, lines, i, path)
|
||||
return pmb.parse._apkbuild.parse_next_attribute(lines, i, path)
|
||||
|
||||
assert func("depends", "pkgname='test'") == (False, None, 0)
|
||||
assert func("no variable here") == (None, None, 0)
|
||||
|
||||
assert func("pkgname", 'pkgname="test"') == (True, "test", 0)
|
||||
assert func("\tno_top_level_var=1") == (None, None, 0)
|
||||
|
||||
assert func("pkgname", "pkgname='test'") == (True, "test", 0)
|
||||
assert func('pkgname="test"') == ("pkgname", "test", 0)
|
||||
|
||||
assert func("pkgname", "pkgname=test") == (True, "test", 0)
|
||||
assert func("pkgname='test'") == ("pkgname", "test", 0)
|
||||
|
||||
assert func("pkgname", 'pkgname="test\n"') == (True, "test", 1)
|
||||
assert func("pkgname=test") == ("pkgname", "test", 0)
|
||||
|
||||
assert func("pkgname", 'pkgname="\ntest\n"') == (True, "test", 2)
|
||||
assert func('pkgname="test\n"') == ("pkgname", "test", 1)
|
||||
|
||||
assert func("pkgname", 'pkgname="test" # random comment\npkgrel=3') == \
|
||||
(True, "test", 0)
|
||||
assert func('pkgname="\ntest\n"') == ("pkgname", "test", 2)
|
||||
|
||||
assert func("pkgver", 'pkgver=2.37 # random comment\npkgrel=3') == \
|
||||
(True, "2.37", 0)
|
||||
assert func('pkgname="test" # random comment\npkgrel=3') == \
|
||||
("pkgname", "test", 0)
|
||||
|
||||
assert func("depends", "depends='\nfirst\nsecond\nthird\n'#") == \
|
||||
(True, "first second third", 4)
|
||||
assert func('pkgver=2.37 # random comment\npkgrel=3') == \
|
||||
("pkgver", "2.37", 0)
|
||||
|
||||
assert func("depends", 'depends="\nfirst\n\tsecond third"') == \
|
||||
(True, "first second third", 2)
|
||||
assert func("depends='\nfirst\nsecond\nthird\n'#") == \
|
||||
("depends", "first second third", 4)
|
||||
|
||||
assert func("depends", 'depends=') == (True, "", 0)
|
||||
assert func('depends="\nfirst\n\tsecond third"') == \
|
||||
("depends", "first second third", 2)
|
||||
|
||||
assert func('depends=') == ("depends", "", 0)
|
||||
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
func("depends", 'depends="\nmissing\nend\nquote\nsign')
|
||||
func('depends="\nmissing\nend\nquote\nsign')
|
||||
assert str(e.value).startswith("Can't find closing")
|
||||
|
||||
with pytest.raises(RuntimeError) as e:
|
||||
func("depends", 'depends="')
|
||||
func('depends="')
|
||||
assert str(e.value).startswith("Can't find closing")
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue