Mirrors/rust
1
0
Fork 0
mirror of https://github.com/rust-lang/rust.git synced 2026-04-26 13:01:27 +03:00
Code Issues Packages Projects Releases Wiki Activity

Reformat Python code with ruff

Browse Source
This commit is contained in:
Jakub Beránek
2024-12-04 23:02:25 +01:00
parent 0b737a163e
commit 536516f949
25 changed files with 1540 additions and 943 deletions
Download Patch File Download Diff File Expand all files Collapse all files
library/core/src/unicode/printable.py
+34 -19
View File
@@ -9,7 +9,8 @@ import csv
import os import os
import subprocess import subprocess
NUM_CODEPOINTS=0x110000 NUM_CODEPOINTS = 0x110000
def to_ranges(iter): def to_ranges(iter):
current = None current = None
@@ -23,11 +24,15 @@ def to_ranges(iter):
if current is not None: if current is not None:
yield tuple(current) yield tuple(current)
def get_escaped(codepoints): def get_escaped(codepoints):
for c in codepoints: for c in codepoints:
if (c.class_ or "Cn") in "Cc Cf Cs Co Cn Zl Zp Zs".split() and c.value != ord(' '): if (c.class_ or "Cn") in "Cc Cf Cs Co Cn Zl Zp Zs".split() and c.value != ord(
" "
):
yield c.value yield c.value
def get_file(f): def get_file(f):
try: try:
return open(os.path.basename(f)) return open(os.path.basename(f))
@@ -35,7 +40,9 @@ def get_file(f):
subprocess.run(["curl", "-O", f], check=True) subprocess.run(["curl", "-O", f], check=True)
return open(os.path.basename(f)) return open(os.path.basename(f))
Codepoint = namedtuple('Codepoint', 'value class_')
Codepoint = namedtuple("Codepoint", "value class_")
def get_codepoints(f): def get_codepoints(f):
r = csv.reader(f, delimiter=";") r = csv.reader(f, delimiter=";")
@@ -66,13 +73,14 @@ def get_codepoints(f):
for c in range(prev_codepoint + 1, NUM_CODEPOINTS): for c in range(prev_codepoint + 1, NUM_CODEPOINTS):
yield Codepoint(c, None) yield Codepoint(c, None)
def compress_singletons(singletons): def compress_singletons(singletons):
uppers = [] # (upper, # items in lowers) uppers = [] # (upper, # items in lowers)
lowers = [] lowers = []
for i in singletons: for i in singletons:
upper = i >> 8 upper = i >> 8
lower = i & 0xff lower = i & 0xFF
if len(uppers) == 0 or uppers[-1][0] != upper: if len(uppers) == 0 or uppers[-1][0] != upper:
uppers.append((upper, 1)) uppers.append((upper, 1))
else: else:
@@ -82,10 +90,11 @@ def compress_singletons(singletons):
return uppers, lowers return uppers, lowers
def compress_normal(normal): def compress_normal(normal):
# lengths 0x00..0x7f are encoded as 00, 01, ..., 7e, 7f # lengths 0x00..0x7f are encoded as 00, 01, ..., 7e, 7f
# lengths 0x80..0x7fff are encoded as 80 80, 80 81, ..., ff fe, ff ff # lengths 0x80..0x7fff are encoded as 80 80, 80 81, ..., ff fe, ff ff
compressed = [] # [truelen, (truelenaux), falselen, (falselenaux)] compressed = [] # [truelen, (truelenaux), falselen, (falselenaux)]
prev_start = 0 prev_start = 0
for start, count in normal: for start, count in normal:
@@ -95,21 +104,22 @@ def compress_normal(normal):
assert truelen < 0x8000 and falselen < 0x8000 assert truelen < 0x8000 and falselen < 0x8000
entry = [] entry = []
if truelen > 0x7f: if truelen > 0x7F:
entry.append(0x80 | (truelen >> 8)) entry.append(0x80 | (truelen >> 8))
entry.append(truelen & 0xff) entry.append(truelen & 0xFF)
else: else:
entry.append(truelen & 0x7f) entry.append(truelen & 0x7F)
if falselen > 0x7f: if falselen > 0x7F:
entry.append(0x80 | (falselen >> 8)) entry.append(0x80 | (falselen >> 8))
entry.append(falselen & 0xff) entry.append(falselen & 0xFF)
else: else:
entry.append(falselen & 0x7f) entry.append(falselen & 0x7F)
compressed.append(entry) compressed.append(entry)
return compressed return compressed
def print_singletons(uppers, lowers, uppersname, lowersname): def print_singletons(uppers, lowers, uppersname, lowersname):
print("#[rustfmt::skip]") print("#[rustfmt::skip]")
print("const {}: &[(u8, u8)] = &[".format(uppersname)) print("const {}: &[(u8, u8)] = &[".format(uppersname))
@@ -119,9 +129,12 @@ def print_singletons(uppers, lowers, uppersname, lowersname):
print("#[rustfmt::skip]") print("#[rustfmt::skip]")
print("const {}: &[u8] = &[".format(lowersname)) print("const {}: &[u8] = &[".format(lowersname))
for i in range(0, len(lowers), 8): for i in range(0, len(lowers), 8):
print(" {}".format(" ".join("{:#04x},".format(x) for x in lowers[i:i+8]))) print(
" {}".format(" ".join("{:#04x},".format(x) for x in lowers[i : i + 8]))
)
print("];") print("];")
def print_normal(normal, normalname): def print_normal(normal, normalname):
print("#[rustfmt::skip]") print("#[rustfmt::skip]")
print("const {}: &[u8] = &[".format(normalname)) print("const {}: &[u8] = &[".format(normalname))
@@ -129,12 +142,13 @@ def print_normal(normal, normalname):
print(" {}".format(" ".join("{:#04x},".format(i) for i in v))) print(" {}".format(" ".join("{:#04x},".format(i) for i in v)))
print("];") print("];")
def main(): def main():
file = get_file("https://www.unicode.org/Public/UNIDATA/UnicodeData.txt") file = get_file("https://www.unicode.org/Public/UNIDATA/UnicodeData.txt")
codepoints = get_codepoints(file) codepoints = get_codepoints(file)
CUTOFF=0x10000 CUTOFF = 0x10000
singletons0 = [] singletons0 = []
singletons1 = [] singletons1 = []
normal0 = [] normal0 = []
@@ -234,10 +248,11 @@ pub(crate) fn is_printable(x: char) -> bool {
}\ }\
""") """)
print() print()
print_singletons(singletons0u, singletons0l, 'SINGLETONS0U', 'SINGLETONS0L') print_singletons(singletons0u, singletons0l, "SINGLETONS0U", "SINGLETONS0L")
print_singletons(singletons1u, singletons1l, 'SINGLETONS1U', 'SINGLETONS1L') print_singletons(singletons1u, singletons1l, "SINGLETONS1U", "SINGLETONS1L")
print_normal(normal0, 'NORMAL0') print_normal(normal0, "NORMAL0")
print_normal(normal1, 'NORMAL1') print_normal(normal1, "NORMAL1")
if __name__ == '__main__':
if __name__ == "__main__":
main() main()
src/bootstrap/bootstrap.py
+394 -271
View File
@@ -19,14 +19,17 @@ try:
except ImportError: except ImportError:
lzma = None lzma = None
def platform_is_win32(): def platform_is_win32():
return sys.platform == 'win32' return sys.platform == "win32"
if platform_is_win32(): if platform_is_win32():
EXE_SUFFIX = ".exe" EXE_SUFFIX = ".exe"
else: else:
EXE_SUFFIX = "" EXE_SUFFIX = ""
def get_cpus(): def get_cpus():
if hasattr(os, "sched_getaffinity"): if hasattr(os, "sched_getaffinity"):
return len(os.sched_getaffinity(0)) return len(os.sched_getaffinity(0))
@@ -51,11 +54,14 @@ def get(base, url, path, checksums, verbose=False):
try: try:
if url not in checksums: if url not in checksums:
raise RuntimeError(("src/stage0 doesn't contain a checksum for {}. " raise RuntimeError(
"Pre-built artifacts might not be available for this " (
"target at this time, see https://doc.rust-lang.org/nightly" "src/stage0 doesn't contain a checksum for {}. "
"/rustc/platform-support.html for more information.") "Pre-built artifacts might not be available for this "
.format(url)) "target at this time, see https://doc.rust-lang.org/nightly"
"/rustc/platform-support.html for more information."
).format(url)
)
sha256 = checksums[url] sha256 = checksums[url]
if os.path.exists(path): if os.path.exists(path):
if verify(path, sha256, False): if verify(path, sha256, False):
@@ -64,8 +70,11 @@ def get(base, url, path, checksums, verbose=False):
return return
else: else:
if verbose: if verbose:
eprint("ignoring already-download file", eprint(
path, "due to failed verification") "ignoring already-download file",
path,
"due to failed verification",
)
os.unlink(path) os.unlink(path)
download(temp_path, "{}/{}".format(base, url), True, verbose) download(temp_path, "{}/{}".format(base, url), True, verbose)
if not verify(temp_path, sha256, verbose): if not verify(temp_path, sha256, verbose):
@@ -79,12 +88,14 @@ def get(base, url, path, checksums, verbose=False):
eprint("removing", temp_path) eprint("removing", temp_path)
os.unlink(temp_path) os.unlink(temp_path)
def curl_version(): def curl_version():
m = re.match(bytes("^curl ([0-9]+)\\.([0-9]+)", "utf8"), require(["curl", "-V"])) m = re.match(bytes("^curl ([0-9]+)\\.([0-9]+)", "utf8"), require(["curl", "-V"]))
if m is None: if m is None:
return (0, 0) return (0, 0)
return (int(m[1]), int(m[2])) return (int(m[1]), int(m[2]))
def download(path, url, probably_big, verbose): def download(path, url, probably_big, verbose):
for _ in range(4): for _ in range(4):
try: try:
@@ -114,32 +125,53 @@ def _download(path, url, probably_big, verbose, exception):
require(["curl", "--version"], exception=platform_is_win32()) require(["curl", "--version"], exception=platform_is_win32())
extra_flags = [] extra_flags = []
if curl_version() > (7, 70): if curl_version() > (7, 70):
extra_flags = [ "--retry-all-errors" ] extra_flags = ["--retry-all-errors"]
# options should be kept in sync with # options should be kept in sync with
# src/bootstrap/src/core/download.rs # src/bootstrap/src/core/download.rs
# for consistency. # for consistency.
# they are also more compreprensivly explained in that file. # they are also more compreprensivly explained in that file.
run(["curl", option] + extra_flags + [ run(
# Follow redirect. ["curl", option]
"--location", + extra_flags
# timeout if speed is < 10 bytes/sec for > 30 seconds + [
"--speed-time", "30", "--speed-limit", "10", # Follow redirect.
# timeout if cannot connect within 30 seconds "--location",
"--connect-timeout", "30", # timeout if speed is < 10 bytes/sec for > 30 seconds
"--output", path, "--speed-time",
"--continue-at", "-", "30",
"--retry", "3", "--show-error", "--remote-time", "--fail", url], "--speed-limit",
"10",
# timeout if cannot connect within 30 seconds
"--connect-timeout",
"30",
"--output",
path,
"--continue-at",
"-",
"--retry",
"3",
"--show-error",
"--remote-time",
"--fail",
url,
],
verbose=verbose, verbose=verbose,
exception=True, # Will raise RuntimeError on failure exception=True, # Will raise RuntimeError on failure
) )
except (subprocess.CalledProcessError, OSError, RuntimeError): except (subprocess.CalledProcessError, OSError, RuntimeError):
# see http://serverfault.com/questions/301128/how-to-download # see http://serverfault.com/questions/301128/how-to-download
script = "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;"
if platform_is_win32(): if platform_is_win32():
run_powershell([ run_powershell(
"[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", [
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(url, path)], script,
"(New-Object System.Net.WebClient).DownloadFile('{}', '{}')".format(
url, path
),
],
verbose=verbose, verbose=verbose,
exception=exception) exception=exception,
)
# Check if the RuntimeError raised by run(curl) should be silenced # Check if the RuntimeError raised by run(curl) should be silenced
elif verbose or exception: elif verbose or exception:
raise raise
@@ -153,9 +185,11 @@ def verify(path, expected, verbose):
found = hashlib.sha256(source.read()).hexdigest() found = hashlib.sha256(source.read()).hexdigest()
verified = found == expected verified = found == expected
if not verified: if not verified:
eprint("invalid checksum:\n" eprint(
" found: {}\n" "invalid checksum:\n" " found: {}\n" " expected: {}".format(
" expected: {}".format(found, expected)) found, expected
)
)
return verified return verified
@@ -170,7 +204,7 @@ def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
name = member.replace(fname + "/", "", 1) name = member.replace(fname + "/", "", 1)
if match is not None and not name.startswith(match): if match is not None and not name.startswith(match):
continue continue
name = name[len(match) + 1:] name = name[len(match) + 1 :]
dst_path = os.path.join(dst, name) dst_path = os.path.join(dst, name)
if verbose: if verbose:
@@ -186,18 +220,18 @@ def unpack(tarball, tarball_suffix, dst, verbose=False, match=None):
def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs): def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
"""Run a child program in a new process""" """Run a child program in a new process"""
if verbose: if verbose:
eprint("running: " + ' '.join(args)) eprint("running: " + " ".join(args))
sys.stdout.flush() sys.stdout.flush()
# Ensure that the .exe is used on Windows just in case a Linux ELF has been # Ensure that the .exe is used on Windows just in case a Linux ELF has been
# compiled in the same directory. # compiled in the same directory.
if os.name == 'nt' and not args[0].endswith('.exe'): if os.name == "nt" and not args[0].endswith(".exe"):
args[0] += '.exe' args[0] += ".exe"
# Use Popen here instead of call() as it apparently allows powershell on # Use Popen here instead of call() as it apparently allows powershell on
# Windows to not lock up waiting for input presumably. # Windows to not lock up waiting for input presumably.
ret = subprocess.Popen(args, **kwargs) ret = subprocess.Popen(args, **kwargs)
code = ret.wait() code = ret.wait()
if code != 0: if code != 0:
err = "failed to run: " + ' '.join(args) err = "failed to run: " + " ".join(args)
if verbose or exception: if verbose or exception:
raise RuntimeError(err) raise RuntimeError(err)
# For most failures, we definitely do want to print this error, or the user will have no # For most failures, we definitely do want to print this error, or the user will have no
@@ -209,30 +243,30 @@ def run(args, verbose=False, exception=False, is_bootstrap=False, **kwargs):
else: else:
sys.exit(err) sys.exit(err)
def run_powershell(script, *args, **kwargs): def run_powershell(script, *args, **kwargs):
"""Run a powershell script""" """Run a powershell script"""
run(["PowerShell.exe", "/nologo", "-Command"] + script, *args, **kwargs) run(["PowerShell.exe", "/nologo", "-Command"] + script, *args, **kwargs)
def require(cmd, exit=True, exception=False): def require(cmd, exit=True, exception=False):
'''Run a command, returning its output. """Run a command, returning its output.
On error, On error,
If `exception` is `True`, raise the error If `exception` is `True`, raise the error
Otherwise If `exit` is `True`, exit the process Otherwise If `exit` is `True`, exit the process
Else return None.''' Else return None."""
try: try:
return subprocess.check_output(cmd).strip() return subprocess.check_output(cmd).strip()
except (subprocess.CalledProcessError, OSError) as exc: except (subprocess.CalledProcessError, OSError) as exc:
if exception: if exception:
raise raise
elif exit: elif exit:
eprint("ERROR: unable to run `{}`: {}".format(' '.join(cmd), exc)) eprint("ERROR: unable to run `{}`: {}".format(" ".join(cmd), exc))
eprint("Please make sure it's installed and in the path.") eprint("Please make sure it's installed and in the path.")
sys.exit(1) sys.exit(1)
return None return None
def format_build_time(duration): def format_build_time(duration):
"""Return a nicer format for build time """Return a nicer format for build time
@@ -252,13 +286,16 @@ def default_build_triple(verbose):
if platform_is_win32(): if platform_is_win32():
try: try:
version = subprocess.check_output(["rustc", "--version", "--verbose"], version = subprocess.check_output(
stderr=subprocess.DEVNULL) ["rustc", "--version", "--verbose"], stderr=subprocess.DEVNULL
)
version = version.decode(default_encoding) version = version.decode(default_encoding)
host = next(x for x in version.split('\n') if x.startswith("host: ")) host = next(x for x in version.split("\n") if x.startswith("host: "))
triple = host.split("host: ")[1] triple = host.split("host: ")[1]
if verbose: if verbose:
eprint("detected default triple {} from pre-installed rustc".format(triple)) eprint(
"detected default triple {} from pre-installed rustc".format(triple)
)
return triple return triple
except Exception as e: except Exception as e:
if verbose: if verbose:
@@ -270,148 +307,149 @@ def default_build_triple(verbose):
# If we do not have `uname`, assume Windows. # If we do not have `uname`, assume Windows.
if uname is None: if uname is None:
return 'x86_64-pc-windows-msvc' return "x86_64-pc-windows-msvc"
kernel, cputype, processor = uname.decode(default_encoding).split(maxsplit=2) kernel, cputype, processor = uname.decode(default_encoding).split(maxsplit=2)
# The goal here is to come up with the same triple as LLVM would, # The goal here is to come up with the same triple as LLVM would,
# at least for the subset of platforms we're willing to target. # at least for the subset of platforms we're willing to target.
kerneltype_mapper = { kerneltype_mapper = {
'Darwin': 'apple-darwin', "Darwin": "apple-darwin",
'DragonFly': 'unknown-dragonfly', "DragonFly": "unknown-dragonfly",
'FreeBSD': 'unknown-freebsd', "FreeBSD": "unknown-freebsd",
'Haiku': 'unknown-haiku', "Haiku": "unknown-haiku",
'NetBSD': 'unknown-netbsd', "NetBSD": "unknown-netbsd",
'OpenBSD': 'unknown-openbsd', "OpenBSD": "unknown-openbsd",
'GNU': 'unknown-hurd', "GNU": "unknown-hurd",
} }
# Consider the direct transformation first and then the special cases # Consider the direct transformation first and then the special cases
if kernel in kerneltype_mapper: if kernel in kerneltype_mapper:
kernel = kerneltype_mapper[kernel] kernel = kerneltype_mapper[kernel]
elif kernel == 'Linux': elif kernel == "Linux":
# Apple doesn't support `-o` so this can't be used in the combined # Apple doesn't support `-o` so this can't be used in the combined
# uname invocation above # uname invocation above
ostype = require(["uname", "-o"], exit=required).decode(default_encoding) ostype = require(["uname", "-o"], exit=required).decode(default_encoding)
if ostype == 'Android': if ostype == "Android":
kernel = 'linux-android' kernel = "linux-android"
else: else:
kernel = 'unknown-linux-gnu' kernel = "unknown-linux-gnu"
elif kernel == 'SunOS': elif kernel == "SunOS":
kernel = 'pc-solaris' kernel = "pc-solaris"
# On Solaris, uname -m will return a machine classification instead # On Solaris, uname -m will return a machine classification instead
# of a cpu type, so uname -p is recommended instead. However, the # of a cpu type, so uname -p is recommended instead. However, the
# output from that option is too generic for our purposes (it will # output from that option is too generic for our purposes (it will
# always emit 'i386' on x86/amd64 systems). As such, isainfo -k # always emit 'i386' on x86/amd64 systems). As such, isainfo -k
# must be used instead. # must be used instead.
cputype = require(['isainfo', '-k']).decode(default_encoding) cputype = require(["isainfo", "-k"]).decode(default_encoding)
# sparc cpus have sun as a target vendor # sparc cpus have sun as a target vendor
if 'sparc' in cputype: if "sparc" in cputype:
kernel = 'sun-solaris' kernel = "sun-solaris"
elif kernel.startswith('MINGW'): elif kernel.startswith("MINGW"):
# msys' `uname` does not print gcc configuration, but prints msys # msys' `uname` does not print gcc configuration, but prints msys
# configuration. so we cannot believe `uname -m`: # configuration. so we cannot believe `uname -m`:
# msys1 is always i686 and msys2 is always x86_64. # msys1 is always i686 and msys2 is always x86_64.
# instead, msys defines $MSYSTEM which is MINGW32 on i686 and # instead, msys defines $MSYSTEM which is MINGW32 on i686 and
# MINGW64 on x86_64. # MINGW64 on x86_64.
kernel = 'pc-windows-gnu' kernel = "pc-windows-gnu"
cputype = 'i686' cputype = "i686"
if os.environ.get('MSYSTEM') == 'MINGW64': if os.environ.get("MSYSTEM") == "MINGW64":
cputype = 'x86_64' cputype = "x86_64"
elif kernel.startswith('MSYS'): elif kernel.startswith("MSYS"):
kernel = 'pc-windows-gnu' kernel = "pc-windows-gnu"
elif kernel.startswith('CYGWIN_NT'): elif kernel.startswith("CYGWIN_NT"):
cputype = 'i686' cputype = "i686"
if kernel.endswith('WOW64'): if kernel.endswith("WOW64"):
cputype = 'x86_64' cputype = "x86_64"
kernel = 'pc-windows-gnu' kernel = "pc-windows-gnu"
elif platform_is_win32(): elif platform_is_win32():
# Some Windows platforms might have a `uname` command that returns a # Some Windows platforms might have a `uname` command that returns a
# non-standard string (e.g. gnuwin32 tools returns `windows32`). In # non-standard string (e.g. gnuwin32 tools returns `windows32`). In
# these cases, fall back to using sys.platform. # these cases, fall back to using sys.platform.
return 'x86_64-pc-windows-msvc' return "x86_64-pc-windows-msvc"
elif kernel == 'AIX': elif kernel == "AIX":
# `uname -m` returns the machine ID rather than machine hardware on AIX, # `uname -m` returns the machine ID rather than machine hardware on AIX,
# so we are unable to use cputype to form triple. AIX 7.2 and # so we are unable to use cputype to form triple. AIX 7.2 and
# above supports 32-bit and 64-bit mode simultaneously and `uname -p` # above supports 32-bit and 64-bit mode simultaneously and `uname -p`
# returns `powerpc`, however we only supports `powerpc64-ibm-aix` in # returns `powerpc`, however we only supports `powerpc64-ibm-aix` in
# rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper # rust on AIX. For above reasons, kerneltype_mapper and cputype_mapper
# are not used to infer AIX's triple. # are not used to infer AIX's triple.
return 'powerpc64-ibm-aix' return "powerpc64-ibm-aix"
else: else:
err = "unknown OS type: {}".format(kernel) err = "unknown OS type: {}".format(kernel)
sys.exit(err) sys.exit(err)
if cputype in ['powerpc', 'riscv'] and kernel == 'unknown-freebsd': if cputype in ["powerpc", "riscv"] and kernel == "unknown-freebsd":
cputype = subprocess.check_output( cputype = (
['uname', '-p']).strip().decode(default_encoding) subprocess.check_output(["uname", "-p"]).strip().decode(default_encoding)
)
cputype_mapper = { cputype_mapper = {
'BePC': 'i686', "BePC": "i686",
'aarch64': 'aarch64', "aarch64": "aarch64",
'aarch64eb': 'aarch64', "aarch64eb": "aarch64",
'amd64': 'x86_64', "amd64": "x86_64",
'arm64': 'aarch64', "arm64": "aarch64",
'i386': 'i686', "i386": "i686",
'i486': 'i686', "i486": "i686",
'i686': 'i686', "i686": "i686",
'i686-AT386': 'i686', "i686-AT386": "i686",
'i786': 'i686', "i786": "i686",
'loongarch64': 'loongarch64', "loongarch64": "loongarch64",
'm68k': 'm68k', "m68k": "m68k",
'csky': 'csky', "csky": "csky",
'powerpc': 'powerpc', "powerpc": "powerpc",
'powerpc64': 'powerpc64', "powerpc64": "powerpc64",
'powerpc64le': 'powerpc64le', "powerpc64le": "powerpc64le",
'ppc': 'powerpc', "ppc": "powerpc",
'ppc64': 'powerpc64', "ppc64": "powerpc64",
'ppc64le': 'powerpc64le', "ppc64le": "powerpc64le",
'riscv64': 'riscv64gc', "riscv64": "riscv64gc",
's390x': 's390x', "s390x": "s390x",
'x64': 'x86_64', "x64": "x86_64",
'x86': 'i686', "x86": "i686",
'x86-64': 'x86_64', "x86-64": "x86_64",
'x86_64': 'x86_64' "x86_64": "x86_64",
} }
# Consider the direct transformation first and then the special cases # Consider the direct transformation first and then the special cases
if cputype in cputype_mapper: if cputype in cputype_mapper:
cputype = cputype_mapper[cputype] cputype = cputype_mapper[cputype]
elif cputype in {'xscale', 'arm'}: elif cputype in {"xscale", "arm"}:
cputype = 'arm' cputype = "arm"
if kernel == 'linux-android': if kernel == "linux-android":
kernel = 'linux-androideabi' kernel = "linux-androideabi"
elif kernel == 'unknown-freebsd': elif kernel == "unknown-freebsd":
cputype = processor cputype = processor
kernel = 'unknown-freebsd' kernel = "unknown-freebsd"
elif cputype == 'armv6l': elif cputype == "armv6l":
cputype = 'arm' cputype = "arm"
if kernel == 'linux-android': if kernel == "linux-android":
kernel = 'linux-androideabi' kernel = "linux-androideabi"
else: else:
kernel += 'eabihf' kernel += "eabihf"
elif cputype in {'armv7l', 'armv8l'}: elif cputype in {"armv7l", "armv8l"}:
cputype = 'armv7' cputype = "armv7"
if kernel == 'linux-android': if kernel == "linux-android":
kernel = 'linux-androideabi' kernel = "linux-androideabi"
else: else:
kernel += 'eabihf' kernel += "eabihf"
elif cputype == 'mips': elif cputype == "mips":
if sys.byteorder == 'big': if sys.byteorder == "big":
cputype = 'mips' cputype = "mips"
elif sys.byteorder == 'little': elif sys.byteorder == "little":
cputype = 'mipsel' cputype = "mipsel"
else: else:
raise ValueError("unknown byteorder: {}".format(sys.byteorder)) raise ValueError("unknown byteorder: {}".format(sys.byteorder))
elif cputype == 'mips64': elif cputype == "mips64":
if sys.byteorder == 'big': if sys.byteorder == "big":
cputype = 'mips64' cputype = "mips64"
elif sys.byteorder == 'little': elif sys.byteorder == "little":
cputype = 'mips64el' cputype = "mips64el"
else: else:
raise ValueError('unknown byteorder: {}'.format(sys.byteorder)) raise ValueError("unknown byteorder: {}".format(sys.byteorder))
# only the n64 ABI is supported, indicate it # only the n64 ABI is supported, indicate it
kernel += 'abi64' kernel += "abi64"
elif cputype == 'sparc' or cputype == 'sparcv9' or cputype == 'sparc64': elif cputype == "sparc" or cputype == "sparcv9" or cputype == "sparc64":
pass pass
else: else:
err = "unknown cpu type: {}".format(cputype) err = "unknown cpu type: {}".format(cputype)
@@ -422,8 +460,8 @@ def default_build_triple(verbose):
@contextlib.contextmanager @contextlib.contextmanager
def output(filepath): def output(filepath):
tmp = filepath + '.tmp' tmp = filepath + ".tmp"
with open(tmp, 'w') as f: with open(tmp, "w") as f:
yield f yield f
try: try:
if os.path.exists(filepath): if os.path.exists(filepath):
@@ -467,6 +505,7 @@ class DownloadInfo:
self.pattern = pattern self.pattern = pattern
self.verbose = verbose self.verbose = verbose
def download_component(download_info): def download_component(download_info):
if not os.path.exists(download_info.tarball_path): if not os.path.exists(download_info.tarball_path):
get( get(
@@ -477,6 +516,7 @@ def download_component(download_info):
verbose=download_info.verbose, verbose=download_info.verbose,
) )
def unpack_component(download_info): def unpack_component(download_info):
unpack( unpack(
download_info.tarball_path, download_info.tarball_path,
@@ -486,26 +526,30 @@ def unpack_component(download_info):
verbose=download_info.verbose, verbose=download_info.verbose,
) )
class FakeArgs: class FakeArgs:
"""Used for unit tests to avoid updating all call sites""" """Used for unit tests to avoid updating all call sites"""
def __init__(self): def __init__(self):
self.build = '' self.build = ""
self.build_dir = '' self.build_dir = ""
self.clean = False self.clean = False
self.verbose = False self.verbose = False
self.json_output = False self.json_output = False
self.color = 'auto' self.color = "auto"
self.warnings = 'default' self.warnings = "default"
class RustBuild(object): class RustBuild(object):
"""Provide all the methods required to build Rust""" """Provide all the methods required to build Rust"""
def __init__(self, config_toml="", args=None): def __init__(self, config_toml="", args=None):
if args is None: if args is None:
args = FakeArgs() args = FakeArgs()
self.git_version = None self.git_version = None
self.nix_deps_dir = None self.nix_deps_dir = None
self._should_fix_bins_and_dylibs = None self._should_fix_bins_and_dylibs = None
self.rust_root = os.path.abspath(os.path.join(__file__, '../../..')) self.rust_root = os.path.abspath(os.path.join(__file__, "../../.."))
self.config_toml = config_toml self.config_toml = config_toml
@@ -515,26 +559,28 @@ class RustBuild(object):
self.color = args.color self.color = args.color
self.warnings = args.warnings self.warnings = args.warnings
config_verbose_count = self.get_toml('verbose', 'build') config_verbose_count = self.get_toml("verbose", "build")
if config_verbose_count is not None: if config_verbose_count is not None:
self.verbose = max(self.verbose, int(config_verbose_count)) self.verbose = max(self.verbose, int(config_verbose_count))
self.use_vendored_sources = self.get_toml('vendor', 'build') == 'true' self.use_vendored_sources = self.get_toml("vendor", "build") == "true"
self.use_locked_deps = self.get_toml('locked-deps', 'build') == 'true' self.use_locked_deps = self.get_toml("locked-deps", "build") == "true"
build_dir = args.build_dir or self.get_toml('build-dir', 'build') or 'build' build_dir = args.build_dir or self.get_toml("build-dir", "build") or "build"
self.build_dir = os.path.abspath(build_dir) self.build_dir = os.path.abspath(build_dir)
self.stage0_data = parse_stage0_file(os.path.join(self.rust_root, "src", "stage0")) self.stage0_data = parse_stage0_file(
self.stage0_compiler = Stage0Toolchain( os.path.join(self.rust_root, "src", "stage0")
self.stage0_data["compiler_date"], )
self.stage0_data["compiler_version"] self.stage0_compiler = Stage0Toolchain(
self.stage0_data["compiler_date"], self.stage0_data["compiler_version"]
)
self.download_url = (
os.getenv("RUSTUP_DIST_SERVER") or self.stage0_data["dist_server"]
) )
self.download_url = os.getenv("RUSTUP_DIST_SERVER") or self.stage0_data["dist_server"]
self.build = args.build or self.build_triple() self.build = args.build or self.build_triple()
def download_toolchain(self): def download_toolchain(self):
"""Fetch the build system for Rust, written in Rust """Fetch the build system for Rust, written in Rust
@@ -550,58 +596,73 @@ class RustBuild(object):
key = self.stage0_compiler.date key = self.stage0_compiler.date
is_outdated = self.program_out_of_date(self.rustc_stamp(), key) is_outdated = self.program_out_of_date(self.rustc_stamp(), key)
need_rustc = self.rustc().startswith(bin_root) and (not os.path.exists(self.rustc()) \ need_rustc = self.rustc().startswith(bin_root) and (
or is_outdated) not os.path.exists(self.rustc()) or is_outdated
need_cargo = self.cargo().startswith(bin_root) and (not os.path.exists(self.cargo()) \ )
or is_outdated) need_cargo = self.cargo().startswith(bin_root) and (
not os.path.exists(self.cargo()) or is_outdated
)
if need_rustc or need_cargo: if need_rustc or need_cargo:
if os.path.exists(bin_root): if os.path.exists(bin_root):
# HACK: On Windows, we can't delete rust-analyzer-proc-macro-server while it's # HACK: On Windows, we can't delete rust-analyzer-proc-macro-server while it's
# running. Kill it. # running. Kill it.
if platform_is_win32(): if platform_is_win32():
print("Killing rust-analyzer-proc-macro-srv before deleting stage0 toolchain") print(
regex = '{}\\\\(host|{})\\\\stage0\\\\libexec'.format( "Killing rust-analyzer-proc-macro-srv before deleting stage0 toolchain"
os.path.basename(self.build_dir), )
self.build regex = "{}\\\\(host|{})\\\\stage0\\\\libexec".format(
os.path.basename(self.build_dir), self.build
) )
script = ( script = (
# NOTE: can't use `taskkill` or `Get-Process -Name` because they error if # NOTE: can't use `taskkill` or `Get-Process -Name` because they error if
# the server isn't running. # the server isn't running.
'Get-Process | ' + "Get-Process | "
'Where-Object {$_.Name -eq "rust-analyzer-proc-macro-srv"} |' + + 'Where-Object {$_.Name -eq "rust-analyzer-proc-macro-srv"} |'
'Where-Object {{$_.Path -match "{}"}} |'.format(regex) + + 'Where-Object {{$_.Path -match "{}"}} |'.format(regex)
'Stop-Process' + "Stop-Process"
) )
run_powershell([script]) run_powershell([script])
shutil.rmtree(bin_root) shutil.rmtree(bin_root)
cache_dst = (self.get_toml('bootstrap-cache-path', 'build') or cache_dst = self.get_toml("bootstrap-cache-path", "build") or os.path.join(
os.path.join(self.build_dir, "cache")) self.build_dir, "cache"
)
rustc_cache = os.path.join(cache_dst, key) rustc_cache = os.path.join(cache_dst, key)
if not os.path.exists(rustc_cache): if not os.path.exists(rustc_cache):
os.makedirs(rustc_cache) os.makedirs(rustc_cache)
tarball_suffix = '.tar.gz' if lzma is None else '.tar.xz' tarball_suffix = ".tar.gz" if lzma is None else ".tar.xz"
toolchain_suffix = "{}-{}{}".format(rustc_channel, self.build, tarball_suffix) toolchain_suffix = "{}-{}{}".format(
rustc_channel, self.build, tarball_suffix
)
tarballs_to_download = [] tarballs_to_download = []
if need_rustc: if need_rustc:
tarballs_to_download.append( tarballs_to_download.append(
("rust-std-{}".format(toolchain_suffix), "rust-std-{}".format(self.build)) (
"rust-std-{}".format(toolchain_suffix),
"rust-std-{}".format(self.build),
)
)
tarballs_to_download.append(
("rustc-{}".format(toolchain_suffix), "rustc")
) )
tarballs_to_download.append(("rustc-{}".format(toolchain_suffix), "rustc"))
if need_cargo: if need_cargo:
tarballs_to_download.append(("cargo-{}".format(toolchain_suffix), "cargo")) tarballs_to_download.append(
("cargo-{}".format(toolchain_suffix), "cargo")
)
tarballs_download_info = [ tarballs_download_info = [
DownloadInfo( DownloadInfo(
base_download_url=self.download_url, base_download_url=self.download_url,
download_path="dist/{}/{}".format(self.stage0_compiler.date, filename), download_path="dist/{}/{}".format(
self.stage0_compiler.date, filename
),
bin_root=self.bin_root(), bin_root=self.bin_root(),
tarball_path=os.path.join(rustc_cache, filename), tarball_path=os.path.join(rustc_cache, filename),
tarball_suffix=tarball_suffix, tarball_suffix=tarball_suffix,
@@ -620,7 +681,11 @@ class RustBuild(object):
# In Python 2.7, Pool cannot be used as a context manager. # In Python 2.7, Pool cannot be used as a context manager.
pool_size = min(len(tarballs_download_info), get_cpus()) pool_size = min(len(tarballs_download_info), get_cpus())
if self.verbose: if self.verbose:
print('Choosing a pool size of', pool_size, 'for the unpacking of the tarballs') print(
"Choosing a pool size of",
pool_size,
"for the unpacking of the tarballs",
)
p = Pool(pool_size) p = Pool(pool_size)
try: try:
# FIXME: A cheap workaround for https://github.com/rust-lang/rust/issues/125578, # FIXME: A cheap workaround for https://github.com/rust-lang/rust/issues/125578,
@@ -639,7 +704,9 @@ class RustBuild(object):
self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root)) self.fix_bin_or_dylib("{}/bin/rustc".format(bin_root))
self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root)) self.fix_bin_or_dylib("{}/bin/rustdoc".format(bin_root))
self.fix_bin_or_dylib("{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root)) self.fix_bin_or_dylib(
"{}/libexec/rust-analyzer-proc-macro-srv".format(bin_root)
)
lib_dir = "{}/lib".format(bin_root) lib_dir = "{}/lib".format(bin_root)
rustlib_bin_dir = "{}/rustlib/{}/bin".format(lib_dir, self.build) rustlib_bin_dir = "{}/rustlib/{}/bin".format(lib_dir, self.build)
self.fix_bin_or_dylib("{}/rust-lld".format(rustlib_bin_dir)) self.fix_bin_or_dylib("{}/rust-lld".format(rustlib_bin_dir))
@@ -667,12 +734,15 @@ class RustBuild(object):
def get_answer(): def get_answer():
default_encoding = sys.getdefaultencoding() default_encoding = sys.getdefaultencoding()
try: try:
ostype = subprocess.check_output( ostype = (
['uname', '-s']).strip().decode(default_encoding) subprocess.check_output(["uname", "-s"])
.strip()
.decode(default_encoding)
)
except subprocess.CalledProcessError: except subprocess.CalledProcessError:
return False return False
except OSError as reason: except OSError as reason:
if getattr(reason, 'winerror', None) is not None: if getattr(reason, "winerror", None) is not None:
return False return False
raise reason raise reason
@@ -690,17 +760,23 @@ class RustBuild(object):
# The latter one does not exist on NixOS when using tmpfs as root. # The latter one does not exist on NixOS when using tmpfs as root.
try: try:
with open("/etc/os-release", "r") as f: with open("/etc/os-release", "r") as f:
is_nixos = any(ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"') is_nixos = any(
for ln in f) ln.strip() in ("ID=nixos", "ID='nixos'", 'ID="nixos"')
for ln in f
)
except FileNotFoundError: except FileNotFoundError:
is_nixos = False is_nixos = False
# If not on NixOS, then warn if user seems to be atop Nix shell # If not on NixOS, then warn if user seems to be atop Nix shell
if not is_nixos: if not is_nixos:
in_nix_shell = os.getenv('IN_NIX_SHELL') in_nix_shell = os.getenv("IN_NIX_SHELL")
if in_nix_shell: if in_nix_shell:
eprint("The IN_NIX_SHELL environment variable is `{}`;".format(in_nix_shell), eprint(
"you may need to set `patch-binaries-for-nix=true` in config.toml") "The IN_NIX_SHELL environment variable is `{}`;".format(
in_nix_shell
),
"you may need to set `patch-binaries-for-nix=true` in config.toml",
)
return is_nixos return is_nixos
@@ -736,7 +812,7 @@ class RustBuild(object):
# zlib: Needed as a system dependency of `libLLVM-*.so`. # zlib: Needed as a system dependency of `libLLVM-*.so`.
# patchelf: Needed for patching ELF binaries (see doc comment above). # patchelf: Needed for patching ELF binaries (see doc comment above).
nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps") nix_deps_dir = "{}/{}".format(self.build_dir, ".nix-deps")
nix_expr = ''' nix_expr = """
with (import <nixpkgs> {}); with (import <nixpkgs> {});
symlinkJoin { symlinkJoin {
name = "rust-stage0-dependencies"; name = "rust-stage0-dependencies";
@@ -746,24 +822,30 @@ class RustBuild(object):
stdenv.cc.bintools stdenv.cc.bintools
]; ];
} }
''' """
try: try:
subprocess.check_output([ subprocess.check_output(
"nix-build", "-E", nix_expr, "-o", nix_deps_dir, [
]) "nix-build",
"-E",
nix_expr,
"-o",
nix_deps_dir,
]
)
except subprocess.CalledProcessError as reason: except subprocess.CalledProcessError as reason:
eprint("WARNING: failed to call nix-build:", reason) eprint("WARNING: failed to call nix-build:", reason)
return return
self.nix_deps_dir = nix_deps_dir self.nix_deps_dir = nix_deps_dir
patchelf = "{}/bin/patchelf".format(nix_deps_dir) patchelf = "{}/bin/patchelf".format(nix_deps_dir)
rpath_entries = [ rpath_entries = [os.path.join(os.path.realpath(nix_deps_dir), "lib")]
os.path.join(os.path.realpath(nix_deps_dir), "lib")
]
patchelf_args = ["--add-rpath", ":".join(rpath_entries)] patchelf_args = ["--add-rpath", ":".join(rpath_entries)]
if ".so" not in fname: if ".so" not in fname:
# Finally, set the correct .interp for binaries # Finally, set the correct .interp for binaries
with open("{}/nix-support/dynamic-linker".format(nix_deps_dir)) as dynamic_linker: with open(
"{}/nix-support/dynamic-linker".format(nix_deps_dir)
) as dynamic_linker:
patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()] patchelf_args += ["--set-interpreter", dynamic_linker.read().rstrip()]
try: try:
@@ -781,13 +863,13 @@ class RustBuild(object):
>>> expected = os.path.join("build", "host", "stage0", ".rustc-stamp") >>> expected = os.path.join("build", "host", "stage0", ".rustc-stamp")
>>> assert rb.rustc_stamp() == expected, rb.rustc_stamp() >>> assert rb.rustc_stamp() == expected, rb.rustc_stamp()
""" """
return os.path.join(self.bin_root(), '.rustc-stamp') return os.path.join(self.bin_root(), ".rustc-stamp")
def program_out_of_date(self, stamp_path, key): def program_out_of_date(self, stamp_path, key):
"""Check if the given program stamp is out of date""" """Check if the given program stamp is out of date"""
if not os.path.exists(stamp_path) or self.clean: if not os.path.exists(stamp_path) or self.clean:
return True return True
with open(stamp_path, 'r') as stamp: with open(stamp_path, "r") as stamp:
return key != stamp.read() return key != stamp.read()
def bin_root(self): def bin_root(self):
@@ -834,11 +916,11 @@ class RustBuild(object):
def get_toml_static(config_toml, key, section=None): def get_toml_static(config_toml, key, section=None):
cur_section = None cur_section = None
for line in config_toml.splitlines(): for line in config_toml.splitlines():
section_match = re.match(r'^\s*\[(.*)\]\s*$', line) section_match = re.match(r"^\s*\[(.*)\]\s*$", line)
if section_match is not None: if section_match is not None:
cur_section = section_match.group(1) cur_section = section_match.group(1)
match = re.match(r'^{}\s*=(.*)$'.format(key), line) match = re.match(r"^{}\s*=(.*)$".format(key), line)
if match is not None: if match is not None:
value = match.group(1) value = match.group(1)
if section is None or section == cur_section: if section is None or section == cur_section:
@@ -847,11 +929,11 @@ class RustBuild(object):
def cargo(self): def cargo(self):
"""Return config path for cargo""" """Return config path for cargo"""
return self.program_config('cargo') return self.program_config("cargo")
def rustc(self): def rustc(self):
"""Return config path for rustc""" """Return config path for rustc"""
return self.program_config('rustc') return self.program_config("rustc")
def program_config(self, program): def program_config(self, program):
"""Return config path for the given program at the given stage """Return config path for the given program at the given stage
@@ -886,12 +968,12 @@ class RustBuild(object):
""" """
start = line.find('"') start = line.find('"')
if start != -1: if start != -1:
end = start + 1 + line[start + 1:].find('"') end = start + 1 + line[start + 1 :].find('"')
return line[start + 1:end] return line[start + 1 : end]
start = line.find('\'') start = line.find("'")
if start != -1: if start != -1:
end = start + 1 + line[start + 1:].find('\'') end = start + 1 + line[start + 1 :].find("'")
return line[start + 1:end] return line[start + 1 : end]
return None return None
def bootstrap_out(self): def bootstrap_out(self):
@@ -941,24 +1023,37 @@ class RustBuild(object):
del env["CARGO_BUILD_TARGET"] del env["CARGO_BUILD_TARGET"]
env["CARGO_TARGET_DIR"] = build_dir env["CARGO_TARGET_DIR"] = build_dir
env["RUSTC"] = self.rustc() env["RUSTC"] = self.rustc()
env["LD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ env["LD_LIBRARY_PATH"] = (
(os.pathsep + env["LD_LIBRARY_PATH"]) \ os.path.join(self.bin_root(), "lib") + (os.pathsep + env["LD_LIBRARY_PATH"])
if "LD_LIBRARY_PATH" in env else "" if "LD_LIBRARY_PATH" in env
env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ else ""
(os.pathsep + env["DYLD_LIBRARY_PATH"]) \ )
if "DYLD_LIBRARY_PATH" in env else "" env["DYLD_LIBRARY_PATH"] = (
env["LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib") + \ os.path.join(self.bin_root(), "lib")
(os.pathsep + env["LIBRARY_PATH"]) \ + (os.pathsep + env["DYLD_LIBRARY_PATH"])
if "LIBRARY_PATH" in env else "" if "DYLD_LIBRARY_PATH" in env
env["LIBPATH"] = os.path.join(self.bin_root(), "lib") + \ else ""
(os.pathsep + env["LIBPATH"]) \ )
if "LIBPATH" in env else "" env["LIBRARY_PATH"] = (
os.path.join(self.bin_root(), "lib") + (os.pathsep + env["LIBRARY_PATH"])
if "LIBRARY_PATH" in env
else ""
)
env["LIBPATH"] = (
os.path.join(self.bin_root(), "lib") + (os.pathsep + env["LIBPATH"])
if "LIBPATH" in env
else ""
)
# Export Stage0 snapshot compiler related env variables # Export Stage0 snapshot compiler related env variables
build_section = "target.{}".format(self.build) build_section = "target.{}".format(self.build)
host_triple_sanitized = self.build.replace("-", "_") host_triple_sanitized = self.build.replace("-", "_")
var_data = { var_data = {
"CC": "cc", "CXX": "cxx", "LD": "linker", "AR": "ar", "RANLIB": "ranlib" "CC": "cc",
"CXX": "cxx",
"LD": "linker",
"AR": "ar",
"RANLIB": "ranlib",
} }
for var_name, toml_key in var_data.items(): for var_name, toml_key in var_data.items():
toml_val = self.get_toml(toml_key, build_section) toml_val = self.get_toml(toml_key, build_section)
@@ -1023,14 +1118,16 @@ class RustBuild(object):
if "RUSTFLAGS_BOOTSTRAP" in env: if "RUSTFLAGS_BOOTSTRAP" in env:
env["RUSTFLAGS"] += " " + env["RUSTFLAGS_BOOTSTRAP"] env["RUSTFLAGS"] += " " + env["RUSTFLAGS_BOOTSTRAP"]
env["PATH"] = os.path.join(self.bin_root(), "bin") + \ env["PATH"] = os.path.join(self.bin_root(), "bin") + os.pathsep + env["PATH"]
os.pathsep + env["PATH"]
if not os.path.isfile(self.cargo()): if not os.path.isfile(self.cargo()):
raise Exception("no cargo executable found at `{}`".format( raise Exception("no cargo executable found at `{}`".format(self.cargo()))
self.cargo())) args = [
args = [self.cargo(), "build", "--manifest-path", self.cargo(),
os.path.join(self.rust_root, "src/bootstrap/Cargo.toml"), "build",
"-Zroot-dir="+self.rust_root] "--manifest-path",
os.path.join(self.rust_root, "src/bootstrap/Cargo.toml"),
"-Zroot-dir=" + self.rust_root,
]
args.extend("--verbose" for _ in range(self.verbose)) args.extend("--verbose" for _ in range(self.verbose))
if self.use_locked_deps: if self.use_locked_deps:
args.append("--locked") args.append("--locked")
@@ -1058,83 +1155,103 @@ class RustBuild(object):
Note that `default_build_triple` is moderately expensive, Note that `default_build_triple` is moderately expensive,
so use `self.build` where possible. so use `self.build` where possible.
""" """
config = self.get_toml('build') config = self.get_toml("build")
return config or default_build_triple(self.verbose) return config or default_build_triple(self.verbose)
def check_vendored_status(self): def check_vendored_status(self):
"""Check that vendoring is configured properly""" """Check that vendoring is configured properly"""
# keep this consistent with the equivalent check in bootstrap: # keep this consistent with the equivalent check in bootstrap:
# https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405 # https://github.com/rust-lang/rust/blob/a8a33cf27166d3eabaffc58ed3799e054af3b0c6/src/bootstrap/lib.rs#L399-L405
if 'SUDO_USER' in os.environ and not self.use_vendored_sources: if "SUDO_USER" in os.environ and not self.use_vendored_sources:
if os.getuid() == 0: if os.getuid() == 0:
self.use_vendored_sources = True self.use_vendored_sources = True
eprint('INFO: looks like you\'re trying to run this command as root') eprint("INFO: looks like you're trying to run this command as root")
eprint(' and so in order to preserve your $HOME this will now') eprint(" and so in order to preserve your $HOME this will now")
eprint(' use vendored sources by default.') eprint(" use vendored sources by default.")
cargo_dir = os.path.join(self.rust_root, '.cargo') cargo_dir = os.path.join(self.rust_root, ".cargo")
url = "https://ci-artifacts.rust-lang.org/rustc-builds/<commit>/rustc-nightly-src.tar.xz"
if self.use_vendored_sources: if self.use_vendored_sources:
vendor_dir = os.path.join(self.rust_root, 'vendor') vendor_dir = os.path.join(self.rust_root, "vendor")
if not os.path.exists(vendor_dir): if not os.path.exists(vendor_dir):
eprint('ERROR: vendoring required, but vendor directory does not exist.') eprint(
eprint(' Run `x.py vendor` to initialize the vendor directory.') "ERROR: vendoring required, but vendor directory does not exist."
eprint(' Alternatively, use the pre-vendored `rustc-src` dist component.') )
eprint(' To get a stable/beta/nightly version, download it from: ') eprint(" Run `x.py vendor` to initialize the vendor directory.")
eprint(' ' eprint(
'https://forge.rust-lang.org/infra/other-installation-methods.html#source-code') " Alternatively, use the pre-vendored `rustc-src` dist component."
eprint(' To get a specific commit version, download it using the below URL,') )
eprint(' replacing <commit> with a specific commit checksum: ') eprint(
eprint(' ' " To get a stable/beta/nightly version, download it from: "
'https://ci-artifacts.rust-lang.org/rustc-builds/<commit>/rustc-nightly-src.tar.xz') )
eprint(' Once you have the source downloaded, place the vendor directory') eprint(
eprint(' from the archive in the root of the rust project.') " "
"https://forge.rust-lang.org/infra/other-installation-methods.html#source-code"
)
eprint(
" To get a specific commit version, download it using the below URL,"
)
eprint(" replacing <commit> with a specific commit checksum: ")
eprint(" ", url)
eprint(
" Once you have the source downloaded, place the vendor directory"
)
eprint(" from the archive in the root of the rust project.")
raise Exception("{} not found".format(vendor_dir)) raise Exception("{} not found".format(vendor_dir))
if not os.path.exists(cargo_dir): if not os.path.exists(cargo_dir):
eprint('ERROR: vendoring required, but .cargo/config does not exist.') eprint("ERROR: vendoring required, but .cargo/config does not exist.")
raise Exception("{} not found".format(cargo_dir)) raise Exception("{} not found".format(cargo_dir))
def parse_args(args): def parse_args(args):
"""Parse the command line arguments that the python script needs.""" """Parse the command line arguments that the python script needs."""
parser = argparse.ArgumentParser(add_help=False) parser = argparse.ArgumentParser(add_help=False)
parser.add_argument('-h', '--help', action='store_true') parser.add_argument("-h", "--help", action="store_true")
parser.add_argument('--config') parser.add_argument("--config")
parser.add_argument('--build-dir') parser.add_argument("--build-dir")
parser.add_argument('--build') parser.add_argument("--build")
parser.add_argument('--color', choices=['always', 'never', 'auto']) parser.add_argument("--color", choices=["always", "never", "auto"])
parser.add_argument('--clean', action='store_true') parser.add_argument("--clean", action="store_true")
parser.add_argument('--json-output', action='store_true') parser.add_argument("--json-output", action="store_true")
parser.add_argument('--warnings', choices=['deny', 'warn', 'default'], default='default') parser.add_argument(
parser.add_argument('-v', '--verbose', action='count', default=0) "--warnings", choices=["deny", "warn", "default"], default="default"
)
parser.add_argument("-v", "--verbose", action="count", default=0)
return parser.parse_known_args(args)[0] return parser.parse_known_args(args)[0]
def parse_stage0_file(path): def parse_stage0_file(path):
result = {} result = {}
with open(path, 'r') as file: with open(path, "r") as file:
for line in file: for line in file:
line = line.strip() line = line.strip()
if line and not line.startswith('#'): if line and not line.startswith("#"):
key, value = line.split('=', 1) key, value = line.split("=", 1)
result[key.strip()] = value.strip() result[key.strip()] = value.strip()
return result return result
def bootstrap(args): def bootstrap(args):
"""Configure, fetch, build and run the initial bootstrap""" """Configure, fetch, build and run the initial bootstrap"""
rust_root = os.path.abspath(os.path.join(__file__, '../../..')) rust_root = os.path.abspath(os.path.join(__file__, "../../.."))
if not os.path.exists(os.path.join(rust_root, '.git')) and \ if not os.path.exists(os.path.join(rust_root, ".git")) and os.path.exists(
os.path.exists(os.path.join(rust_root, '.github')): os.path.join(rust_root, ".github")
eprint("warn: Looks like you are trying to bootstrap Rust from a source that is neither a " ):
"git clone nor distributed tarball.\nThis build may fail due to missing submodules " eprint(
"unless you put them in place manually.") "warn: Looks like you are trying to bootstrap Rust from a source that is neither a "
"git clone nor distributed tarball.\nThis build may fail due to missing submodules "
"unless you put them in place manually."
)
# Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`, # Read from `--config`, then `RUST_BOOTSTRAP_CONFIG`, then `./config.toml`,
# then `config.toml` in the root directory. # then `config.toml` in the root directory.
toml_path = args.config or os.getenv('RUST_BOOTSTRAP_CONFIG') toml_path = args.config or os.getenv("RUST_BOOTSTRAP_CONFIG")
using_default_path = toml_path is None using_default_path = toml_path is None
if using_default_path: if using_default_path:
toml_path = 'config.toml' toml_path = "config.toml"
if not os.path.exists(toml_path): if not os.path.exists(toml_path):
toml_path = os.path.join(rust_root, toml_path) toml_path = os.path.join(rust_root, toml_path)
@@ -1144,23 +1261,23 @@ def bootstrap(args):
with open(toml_path) as config: with open(toml_path) as config:
config_toml = config.read() config_toml = config.read()
else: else:
config_toml = '' config_toml = ""
profile = RustBuild.get_toml_static(config_toml, 'profile') profile = RustBuild.get_toml_static(config_toml, "profile")
if profile is not None: if profile is not None:
# Allows creating alias for profile names, allowing # Allows creating alias for profile names, allowing
# profiles to be renamed while maintaining back compatibility # profiles to be renamed while maintaining back compatibility
# Keep in sync with `profile_aliases` in config.rs # Keep in sync with `profile_aliases` in config.rs
profile_aliases = { profile_aliases = {"user": "dist"}
"user": "dist" include_file = "config.{}.toml".format(profile_aliases.get(profile) or profile)
} include_dir = os.path.join(rust_root, "src", "bootstrap", "defaults")
include_file = 'config.{}.toml'.format(profile_aliases.get(profile) or profile)
include_dir = os.path.join(rust_root, 'src', 'bootstrap', 'defaults')
include_path = os.path.join(include_dir, include_file) include_path = os.path.join(include_dir, include_file)
if not os.path.exists(include_path): if not os.path.exists(include_path):
raise Exception("Unrecognized config profile '{}'. Check src/bootstrap/defaults" raise Exception(
" for available options.".format(profile)) "Unrecognized config profile '{}'. Check src/bootstrap/defaults"
" for available options.".format(profile)
)
# HACK: This works because `self.get_toml()` returns the first match it finds for a # HACK: This works because `self.get_toml()` returns the first match it finds for a
# specific key, so appending our defaults at the end allows the user to override them # specific key, so appending our defaults at the end allows the user to override them
@@ -1194,8 +1311,8 @@ def main():
start_time = time() start_time = time()
# x.py help <cmd> ... # x.py help <cmd> ...
if len(sys.argv) > 1 and sys.argv[1] == 'help': if len(sys.argv) > 1 and sys.argv[1] == "help":
sys.argv[1] = '-h' sys.argv[1] = "-h"
args = parse_args(sys.argv) args = parse_args(sys.argv)
help_triggered = args.help or len(sys.argv) == 1 help_triggered = args.help or len(sys.argv) == 1
@@ -1205,14 +1322,15 @@ def main():
if help_triggered: if help_triggered:
eprint( eprint(
"INFO: Downloading and building bootstrap before processing --help command.\n" "INFO: Downloading and building bootstrap before processing --help command.\n"
" See src/bootstrap/README.md for help with common commands.") " See src/bootstrap/README.md for help with common commands."
)
exit_code = 0 exit_code = 0
success_word = "successfully" success_word = "successfully"
try: try:
bootstrap(args) bootstrap(args)
except (SystemExit, KeyboardInterrupt) as error: except (SystemExit, KeyboardInterrupt) as error:
if hasattr(error, 'code') and isinstance(error.code, int): if hasattr(error, "code") and isinstance(error.code, int):
exit_code = error.code exit_code = error.code
else: else:
exit_code = 1 exit_code = 1
@@ -1220,9 +1338,14 @@ def main():
success_word = "unsuccessfully" success_word = "unsuccessfully"
if not help_triggered: if not help_triggered:
eprint("Build completed", success_word, "in", format_build_time(time() - start_time)) eprint(
"Build completed",
success_word,
"in",
format_build_time(time() - start_time),
)
sys.exit(exit_code) sys.exit(exit_code)
if __name__ == '__main__': if __name__ == "__main__":
main() main()
src/bootstrap/bootstrap_test.py
+29 -13
View File
@@ -16,8 +16,9 @@ from shutil import rmtree
bootstrap_dir = os.path.dirname(os.path.abspath(__file__)) bootstrap_dir = os.path.dirname(os.path.abspath(__file__))
# For the import below, have Python search in src/bootstrap first. # For the import below, have Python search in src/bootstrap first.
sys.path.insert(0, bootstrap_dir) sys.path.insert(0, bootstrap_dir)
import bootstrap # noqa: E402 import bootstrap # noqa: E402
import configure # noqa: E402 import configure # noqa: E402
def serialize_and_parse(configure_args, bootstrap_args=None): def serialize_and_parse(configure_args, bootstrap_args=None):
from io import StringIO from io import StringIO
@@ -32,15 +33,20 @@ def serialize_and_parse(configure_args, bootstrap_args=None):
try: try:
import tomllib import tomllib
# Verify this is actually valid TOML. # Verify this is actually valid TOML.
tomllib.loads(build.config_toml) tomllib.loads(build.config_toml)
except ImportError: except ImportError:
print("WARNING: skipping TOML validation, need at least python 3.11", file=sys.stderr) print(
"WARNING: skipping TOML validation, need at least python 3.11",
file=sys.stderr,
)
return build return build
class VerifyTestCase(unittest.TestCase): class VerifyTestCase(unittest.TestCase):
"""Test Case for verify""" """Test Case for verify"""
def setUp(self): def setUp(self):
self.container = tempfile.mkdtemp() self.container = tempfile.mkdtemp()
self.src = os.path.join(self.container, "src.txt") self.src = os.path.join(self.container, "src.txt")
@@ -68,14 +74,14 @@ class VerifyTestCase(unittest.TestCase):
class ProgramOutOfDate(unittest.TestCase): class ProgramOutOfDate(unittest.TestCase):
"""Test if a program is out of date""" """Test if a program is out of date"""
def setUp(self): def setUp(self):
self.container = tempfile.mkdtemp() self.container = tempfile.mkdtemp()
os.mkdir(os.path.join(self.container, "stage0")) os.mkdir(os.path.join(self.container, "stage0"))
self.build = bootstrap.RustBuild() self.build = bootstrap.RustBuild()
self.build.date = "2017-06-15" self.build.date = "2017-06-15"
self.build.build_dir = self.container self.build.build_dir = self.container
self.rustc_stamp_path = os.path.join(self.container, "stage0", self.rustc_stamp_path = os.path.join(self.container, "stage0", ".rustc-stamp")
".rustc-stamp")
self.key = self.build.date + str(None) self.key = self.build.date + str(None)
def tearDown(self): def tearDown(self):
@@ -97,11 +103,14 @@ class ProgramOutOfDate(unittest.TestCase):
"""Return False both dates match""" """Return False both dates match"""
with open(self.rustc_stamp_path, "w") as rustc_stamp: with open(self.rustc_stamp_path, "w") as rustc_stamp:
rustc_stamp.write("2017-06-15None") rustc_stamp.write("2017-06-15None")
self.assertFalse(self.build.program_out_of_date(self.rustc_stamp_path, self.key)) self.assertFalse(
self.build.program_out_of_date(self.rustc_stamp_path, self.key)
)
class ParseArgsInConfigure(unittest.TestCase): class ParseArgsInConfigure(unittest.TestCase):
"""Test if `parse_args` function in `configure.py` works properly""" """Test if `parse_args` function in `configure.py` works properly"""
@patch("configure.err") @patch("configure.err")
def test_unknown_args(self, err): def test_unknown_args(self, err):
# It should be print an error message if the argument doesn't start with '--' # It should be print an error message if the argument doesn't start with '--'
@@ -148,28 +157,35 @@ class ParseArgsInConfigure(unittest.TestCase):
class GenerateAndParseConfig(unittest.TestCase): class GenerateAndParseConfig(unittest.TestCase):
"""Test that we can serialize and deserialize a config.toml file""" """Test that we can serialize and deserialize a config.toml file"""
def test_no_args(self): def test_no_args(self):
build = serialize_and_parse([]) build = serialize_and_parse([])
self.assertEqual(build.get_toml("profile"), 'dist') self.assertEqual(build.get_toml("profile"), "dist")
self.assertIsNone(build.get_toml("llvm.download-ci-llvm")) self.assertIsNone(build.get_toml("llvm.download-ci-llvm"))
def test_set_section(self): def test_set_section(self):
build = serialize_and_parse(["--set", "llvm.download-ci-llvm"]) build = serialize_and_parse(["--set", "llvm.download-ci-llvm"])
self.assertEqual(build.get_toml("download-ci-llvm", section="llvm"), 'true') self.assertEqual(build.get_toml("download-ci-llvm", section="llvm"), "true")
def test_set_target(self): def test_set_target(self):
build = serialize_and_parse(["--set", "target.x86_64-unknown-linux-gnu.cc=gcc"]) build = serialize_and_parse(["--set", "target.x86_64-unknown-linux-gnu.cc=gcc"])
self.assertEqual(build.get_toml("cc", section="target.x86_64-unknown-linux-gnu"), 'gcc') self.assertEqual(
build.get_toml("cc", section="target.x86_64-unknown-linux-gnu"), "gcc"
)
def test_set_top_level(self): def test_set_top_level(self):
build = serialize_and_parse(["--set", "profile=compiler"]) build = serialize_and_parse(["--set", "profile=compiler"])
self.assertEqual(build.get_toml("profile"), 'compiler') self.assertEqual(build.get_toml("profile"), "compiler")
def test_set_codegen_backends(self): def test_set_codegen_backends(self):
build = serialize_and_parse(["--set", "rust.codegen-backends=cranelift"]) build = serialize_and_parse(["--set", "rust.codegen-backends=cranelift"])
self.assertNotEqual(build.config_toml.find("codegen-backends = ['cranelift']"), -1) self.assertNotEqual(
build.config_toml.find("codegen-backends = ['cranelift']"), -1
)
build = serialize_and_parse(["--set", "rust.codegen-backends=cranelift,llvm"]) build = serialize_and_parse(["--set", "rust.codegen-backends=cranelift,llvm"])
self.assertNotEqual(build.config_toml.find("codegen-backends = ['cranelift', 'llvm']"), -1) self.assertNotEqual(
build.config_toml.find("codegen-backends = ['cranelift', 'llvm']"), -1
)
build = serialize_and_parse(["--enable-full-tools"]) build = serialize_and_parse(["--enable-full-tools"])
self.assertNotEqual(build.config_toml.find("codegen-backends = ['llvm']"), -1) self.assertNotEqual(build.config_toml.find("codegen-backends = ['llvm']"), -1)
@@ -223,7 +239,7 @@ class BuildBootstrap(unittest.TestCase):
self.assertTrue("--timings" in args) self.assertTrue("--timings" in args)
def test_warnings(self): def test_warnings(self):
for toml_warnings in ['false', 'true', None]: for toml_warnings in ["false", "true", None]:
configure_args = [] configure_args = []
if toml_warnings is not None: if toml_warnings is not None:
configure_args = ["--set", "rust.deny-warnings=" + toml_warnings] configure_args = ["--set", "rust.deny-warnings=" + toml_warnings]
src/bootstrap/configure.py
+350 -169
View File
@@ -6,11 +6,12 @@ from __future__ import absolute_import, division, print_function
import shlex import shlex
import sys import sys
import os import os
rust_dir = os.path.dirname(os.path.abspath(__file__)) rust_dir = os.path.dirname(os.path.abspath(__file__))
rust_dir = os.path.dirname(rust_dir) rust_dir = os.path.dirname(rust_dir)
rust_dir = os.path.dirname(rust_dir) rust_dir = os.path.dirname(rust_dir)
sys.path.append(os.path.join(rust_dir, "src", "bootstrap")) sys.path.append(os.path.join(rust_dir, "src", "bootstrap"))
import bootstrap # noqa: E402 import bootstrap # noqa: E402
class Option(object): class Option(object):
@@ -32,26 +33,62 @@ def v(*args):
options.append(Option(*args, value=True)) options.append(Option(*args, value=True))
o("debug", "rust.debug", "enables debugging environment; does not affect optimization of bootstrapped code") o(
"debug",
"rust.debug",
"enables debugging environment; does not affect optimization of bootstrapped code",
)
o("docs", "build.docs", "build standard library documentation") o("docs", "build.docs", "build standard library documentation")
o("compiler-docs", "build.compiler-docs", "build compiler documentation") o("compiler-docs", "build.compiler-docs", "build compiler documentation")
o("optimize-tests", "rust.optimize-tests", "build tests with optimizations") o("optimize-tests", "rust.optimize-tests", "build tests with optimizations")
o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests") o("verbose-tests", "rust.verbose-tests", "enable verbose output when running tests")
o("ccache", "llvm.ccache", "invoke gcc/clang via ccache to reuse object files between builds") o(
"ccache",
"llvm.ccache",
"invoke gcc/clang via ccache to reuse object files between builds",
)
o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds") o("sccache", None, "invoke gcc/clang via sccache to reuse object files between builds")
o("local-rust", None, "use an installed rustc rather than downloading a snapshot") o("local-rust", None, "use an installed rustc rather than downloading a snapshot")
v("local-rust-root", None, "set prefix for local rust binary") v("local-rust-root", None, "set prefix for local rust binary")
o("local-rebuild", "build.local-rebuild", "assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version") o(
o("llvm-static-stdcpp", "llvm.static-libstdcpp", "statically link to libstdc++ for LLVM") "local-rebuild",
o("llvm-link-shared", "llvm.link-shared", "prefer shared linking to LLVM (llvm-config --link-shared)") "build.local-rebuild",
"assume local-rust matches the current version, for rebuilds; implies local-rust, and is implied if local-rust already matches the current version",
)
o(
"llvm-static-stdcpp",
"llvm.static-libstdcpp",
"statically link to libstdc++ for LLVM",
)
o(
"llvm-link-shared",
"llvm.link-shared",
"prefer shared linking to LLVM (llvm-config --link-shared)",
)
o("rpath", "rust.rpath", "build rpaths into rustc itself") o("rpath", "rust.rpath", "build rpaths into rustc itself")
o("codegen-tests", "rust.codegen-tests", "run the tests/codegen tests") o("codegen-tests", "rust.codegen-tests", "run the tests/codegen tests")
o("ninja", "llvm.ninja", "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)") o(
"ninja",
"llvm.ninja",
"build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)",
)
o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date") o("locked-deps", "build.locked-deps", "force Cargo.lock to be up to date")
o("vendor", "build.vendor", "enable usage of vendored Rust crates") o("vendor", "build.vendor", "enable usage of vendored Rust crates")
o("sanitizers", "build.sanitizers", "build the sanitizer runtimes (asan, dfsan, lsan, msan, tsan, hwasan)") o(
o("dist-src", "rust.dist-src", "when building tarballs enables building a source tarball") "sanitizers",
o("cargo-native-static", "build.cargo-native-static", "static native libraries in cargo") "build.sanitizers",
"build the sanitizer runtimes (asan, dfsan, lsan, msan, tsan, hwasan)",
)
o(
"dist-src",
"rust.dist-src",
"when building tarballs enables building a source tarball",
)
o(
"cargo-native-static",
"build.cargo-native-static",
"static native libraries in cargo",
)
o("profiler", "build.profiler", "build the profiler runtime") o("profiler", "build.profiler", "build the profiler runtime")
o("full-tools", None, "enable all tools") o("full-tools", None, "enable all tools")
o("lld", "rust.lld", "build lld") o("lld", "rust.lld", "build lld")
@@ -59,7 +96,11 @@ o("llvm-bitcode-linker", "rust.llvm-bitcode-linker", "build llvm bitcode linker"
o("clang", "llvm.clang", "build clang") o("clang", "llvm.clang", "build clang")
o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++") o("use-libcxx", "llvm.use-libcxx", "build LLVM with libc++")
o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard") o("control-flow-guard", "rust.control-flow-guard", "Enable Control Flow Guard")
o("patch-binaries-for-nix", "build.patch-binaries-for-nix", "whether patch binaries for usage with Nix toolchains") o(
"patch-binaries-for-nix",
"build.patch-binaries-for-nix",
"whether patch binaries for usage with Nix toolchains",
)
o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0") o("new-symbol-mangling", "rust.new-symbol-mangling", "use symbol-mangling-version v0")
v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags") v("llvm-cflags", "llvm.cflags", "build LLVM with these extra compiler flags")
@@ -76,16 +117,48 @@ o("llvm-enzyme", "llvm.enzyme", "build LLVM with enzyme")
o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support") o("llvm-offload", "llvm.offload", "build LLVM with gpu offload support")
o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface") o("llvm-plugins", "llvm.plugins", "build LLVM with plugin interface")
o("debug-assertions", "rust.debug-assertions", "build with debugging assertions") o("debug-assertions", "rust.debug-assertions", "build with debugging assertions")
o("debug-assertions-std", "rust.debug-assertions-std", "build the standard library with debugging assertions") o(
"debug-assertions-std",
"rust.debug-assertions-std",
"build the standard library with debugging assertions",
)
o("overflow-checks", "rust.overflow-checks", "build with overflow checks") o("overflow-checks", "rust.overflow-checks", "build with overflow checks")
o("overflow-checks-std", "rust.overflow-checks-std", "build the standard library with overflow checks") o(
o("llvm-release-debuginfo", "llvm.release-debuginfo", "build LLVM with debugger metadata") "overflow-checks-std",
"rust.overflow-checks-std",
"build the standard library with overflow checks",
)
o(
"llvm-release-debuginfo",
"llvm.release-debuginfo",
"build LLVM with debugger metadata",
)
v("debuginfo-level", "rust.debuginfo-level", "debuginfo level for Rust code") v("debuginfo-level", "rust.debuginfo-level", "debuginfo level for Rust code")
v("debuginfo-level-rustc", "rust.debuginfo-level-rustc", "debuginfo level for the compiler") v(
v("debuginfo-level-std", "rust.debuginfo-level-std", "debuginfo level for the standard library") "debuginfo-level-rustc",
v("debuginfo-level-tools", "rust.debuginfo-level-tools", "debuginfo level for the tools") "rust.debuginfo-level-rustc",
v("debuginfo-level-tests", "rust.debuginfo-level-tests", "debuginfo level for the test suites run with compiletest") "debuginfo level for the compiler",
v("save-toolstates", "rust.save-toolstates", "save build and test status of external tools into this file") )
v(
"debuginfo-level-std",
"rust.debuginfo-level-std",
"debuginfo level for the standard library",
)
v(
"debuginfo-level-tools",
"rust.debuginfo-level-tools",
"debuginfo level for the tools",
)
v(
"debuginfo-level-tests",
"rust.debuginfo-level-tests",
"debuginfo level for the test suites run with compiletest",
)
v(
"save-toolstates",
"rust.save-toolstates",
"save build and test status of external tools into this file",
)
v("prefix", "install.prefix", "set installation prefix") v("prefix", "install.prefix", "set installation prefix")
v("localstatedir", "install.localstatedir", "local state directory") v("localstatedir", "install.localstatedir", "local state directory")
@@ -102,50 +175,117 @@ v("llvm-config", None, "set path to llvm-config")
v("llvm-filecheck", None, "set path to LLVM's FileCheck utility") v("llvm-filecheck", None, "set path to LLVM's FileCheck utility")
v("python", "build.python", "set path to python") v("python", "build.python", "set path to python")
v("android-ndk", "build.android-ndk", "set path to Android NDK") v("android-ndk", "build.android-ndk", "set path to Android NDK")
v("musl-root", "target.x86_64-unknown-linux-musl.musl-root", v(
"MUSL root installation directory (deprecated)") "musl-root",
v("musl-root-x86_64", "target.x86_64-unknown-linux-musl.musl-root", "target.x86_64-unknown-linux-musl.musl-root",
"x86_64-unknown-linux-musl install directory") "MUSL root installation directory (deprecated)",
v("musl-root-i586", "target.i586-unknown-linux-musl.musl-root", )
"i586-unknown-linux-musl install directory") v(
v("musl-root-i686", "target.i686-unknown-linux-musl.musl-root", "musl-root-x86_64",
"i686-unknown-linux-musl install directory") "target.x86_64-unknown-linux-musl.musl-root",
v("musl-root-arm", "target.arm-unknown-linux-musleabi.musl-root", "x86_64-unknown-linux-musl install directory",
"arm-unknown-linux-musleabi install directory") )
v("musl-root-armhf", "target.arm-unknown-linux-musleabihf.musl-root", v(
"arm-unknown-linux-musleabihf install directory") "musl-root-i586",
v("musl-root-armv5te", "target.armv5te-unknown-linux-musleabi.musl-root", "target.i586-unknown-linux-musl.musl-root",
"armv5te-unknown-linux-musleabi install directory") "i586-unknown-linux-musl install directory",
v("musl-root-armv7", "target.armv7-unknown-linux-musleabi.musl-root", )
"armv7-unknown-linux-musleabi install directory") v(
v("musl-root-armv7hf", "target.armv7-unknown-linux-musleabihf.musl-root", "musl-root-i686",
"armv7-unknown-linux-musleabihf install directory") "target.i686-unknown-linux-musl.musl-root",
v("musl-root-aarch64", "target.aarch64-unknown-linux-musl.musl-root", "i686-unknown-linux-musl install directory",
"aarch64-unknown-linux-musl install directory") )
v("musl-root-mips", "target.mips-unknown-linux-musl.musl-root", v(
"mips-unknown-linux-musl install directory") "musl-root-arm",
v("musl-root-mipsel", "target.mipsel-unknown-linux-musl.musl-root", "target.arm-unknown-linux-musleabi.musl-root",
"mipsel-unknown-linux-musl install directory") "arm-unknown-linux-musleabi install directory",
v("musl-root-mips64", "target.mips64-unknown-linux-muslabi64.musl-root", )
"mips64-unknown-linux-muslabi64 install directory") v(
v("musl-root-mips64el", "target.mips64el-unknown-linux-muslabi64.musl-root", "musl-root-armhf",
"mips64el-unknown-linux-muslabi64 install directory") "target.arm-unknown-linux-musleabihf.musl-root",
v("musl-root-riscv32gc", "target.riscv32gc-unknown-linux-musl.musl-root", "arm-unknown-linux-musleabihf install directory",
"riscv32gc-unknown-linux-musl install directory") )
v("musl-root-riscv64gc", "target.riscv64gc-unknown-linux-musl.musl-root", v(
"riscv64gc-unknown-linux-musl install directory") "musl-root-armv5te",
v("musl-root-loongarch64", "target.loongarch64-unknown-linux-musl.musl-root", "target.armv5te-unknown-linux-musleabi.musl-root",
"loongarch64-unknown-linux-musl install directory") "armv5te-unknown-linux-musleabi install directory",
v("qemu-armhf-rootfs", "target.arm-unknown-linux-gnueabihf.qemu-rootfs", )
"rootfs in qemu testing, you probably don't want to use this") v(
v("qemu-aarch64-rootfs", "target.aarch64-unknown-linux-gnu.qemu-rootfs", "musl-root-armv7",
"rootfs in qemu testing, you probably don't want to use this") "target.armv7-unknown-linux-musleabi.musl-root",
v("qemu-riscv64-rootfs", "target.riscv64gc-unknown-linux-gnu.qemu-rootfs", "armv7-unknown-linux-musleabi install directory",
"rootfs in qemu testing, you probably don't want to use this") )
v("experimental-targets", "llvm.experimental-targets", v(
"experimental LLVM targets to build") "musl-root-armv7hf",
"target.armv7-unknown-linux-musleabihf.musl-root",
"armv7-unknown-linux-musleabihf install directory",
)
v(
"musl-root-aarch64",
"target.aarch64-unknown-linux-musl.musl-root",
"aarch64-unknown-linux-musl install directory",
)
v(
"musl-root-mips",
"target.mips-unknown-linux-musl.musl-root",
"mips-unknown-linux-musl install directory",
)
v(
"musl-root-mipsel",
"target.mipsel-unknown-linux-musl.musl-root",
"mipsel-unknown-linux-musl install directory",
)
v(
"musl-root-mips64",
"target.mips64-unknown-linux-muslabi64.musl-root",
"mips64-unknown-linux-muslabi64 install directory",
)
v(
"musl-root-mips64el",
"target.mips64el-unknown-linux-muslabi64.musl-root",
"mips64el-unknown-linux-muslabi64 install directory",
)
v(
"musl-root-riscv32gc",
"target.riscv32gc-unknown-linux-musl.musl-root",
"riscv32gc-unknown-linux-musl install directory",
)
v(
"musl-root-riscv64gc",
"target.riscv64gc-unknown-linux-musl.musl-root",
"riscv64gc-unknown-linux-musl install directory",
)
v(
"musl-root-loongarch64",
"target.loongarch64-unknown-linux-musl.musl-root",
"loongarch64-unknown-linux-musl install directory",
)
v(
"qemu-armhf-rootfs",
"target.arm-unknown-linux-gnueabihf.qemu-rootfs",
"rootfs in qemu testing, you probably don't want to use this",
)
v(
"qemu-aarch64-rootfs",
"target.aarch64-unknown-linux-gnu.qemu-rootfs",
"rootfs in qemu testing, you probably don't want to use this",
)
v(
"qemu-riscv64-rootfs",
"target.riscv64gc-unknown-linux-gnu.qemu-rootfs",
"rootfs in qemu testing, you probably don't want to use this",
)
v(
"experimental-targets",
"llvm.experimental-targets",
"experimental LLVM targets to build",
)
v("release-channel", "rust.channel", "the name of the release channel to build") v("release-channel", "rust.channel", "the name of the release channel to build")
v("release-description", "rust.description", "optional descriptive string for version output") v(
"release-description",
"rust.description",
"optional descriptive string for version output",
)
v("dist-compression-formats", None, "List of compression formats to use") v("dist-compression-formats", None, "List of compression formats to use")
# Used on systems where "cc" is unavailable # Used on systems where "cc" is unavailable
@@ -154,7 +294,11 @@ v("default-linker", "rust.default-linker", "the default linker")
# Many of these are saved below during the "writing configuration" step # Many of these are saved below during the "writing configuration" step
# (others are conditionally saved). # (others are conditionally saved).
o("manage-submodules", "build.submodules", "let the build manage the git submodules") o("manage-submodules", "build.submodules", "let the build manage the git submodules")
o("full-bootstrap", "build.full-bootstrap", "build three compilers instead of two (not recommended except for testing reproducible builds)") o(
"full-bootstrap",
"build.full-bootstrap",
"build three compilers instead of two (not recommended except for testing reproducible builds)",
)
o("extended", "build.extended", "build an extended rust tool set") o("extended", "build.extended", "build an extended rust tool set")
v("bootstrap-cache-path", None, "use provided path for the bootstrap cache") v("bootstrap-cache-path", None, "use provided path for the bootstrap cache")
@@ -165,8 +309,16 @@ v("host", None, "List of GNUs ./configure syntax LLVM host triples")
v("target", None, "List of GNUs ./configure syntax LLVM target triples") v("target", None, "List of GNUs ./configure syntax LLVM target triples")
# Options specific to this configure script # Options specific to this configure script
o("option-checking", None, "complain about unrecognized options in this configure script") o(
o("verbose-configure", None, "don't truncate options when printing them in this configure script") "option-checking",
None,
"complain about unrecognized options in this configure script",
)
o(
"verbose-configure",
None,
"don't truncate options when printing them in this configure script",
)
v("set", None, "set arbitrary key/value pairs in TOML configuration") v("set", None, "set arbitrary key/value pairs in TOML configuration")
@@ -178,39 +330,42 @@ def err(msg):
print("\nconfigure: ERROR: " + msg + "\n") print("\nconfigure: ERROR: " + msg + "\n")
sys.exit(1) sys.exit(1)
def is_value_list(key): def is_value_list(key):
for option in options: for option in options:
if option.name == key and option.desc.startswith('List of'): if option.name == key and option.desc.startswith("List of"):
return True return True
return False return False
if '--help' in sys.argv or '-h' in sys.argv:
print('Usage: ./configure [options]') if "--help" in sys.argv or "-h" in sys.argv:
print('') print("Usage: ./configure [options]")
print('Options') print("")
print("Options")
for option in options: for option in options:
if 'android' in option.name: if "android" in option.name:
# no one needs to know about these obscure options # no one needs to know about these obscure options
continue continue
if option.value: if option.value:
print('\t{:30} {}'.format('--{}=VAL'.format(option.name), option.desc)) print("\t{:30} {}".format("--{}=VAL".format(option.name), option.desc))
else: else:
print('\t--enable-{:25} OR --disable-{}'.format(option.name, option.name)) print("\t--enable-{:25} OR --disable-{}".format(option.name, option.name))
print('\t\t' + option.desc) print("\t\t" + option.desc)
print('') print("")
print('This configure script is a thin configuration shim over the true') print("This configure script is a thin configuration shim over the true")
print('configuration system, `config.toml`. You can explore the comments') print("configuration system, `config.toml`. You can explore the comments")
print('in `config.example.toml` next to this configure script to see') print("in `config.example.toml` next to this configure script to see")
print('more information about what each option is. Additionally you can') print("more information about what each option is. Additionally you can")
print('pass `--set` as an argument to set arbitrary key/value pairs') print("pass `--set` as an argument to set arbitrary key/value pairs")
print('in the TOML configuration if desired') print("in the TOML configuration if desired")
print('') print("")
print('Also note that all options which take `--enable` can similarly') print("Also note that all options which take `--enable` can similarly")
print('be passed with `--disable-foo` to forcibly disable the option') print("be passed with `--disable-foo` to forcibly disable the option")
sys.exit(0) sys.exit(0)
VERBOSE = False VERBOSE = False
# Parse all command line arguments into one of these three lists, handling # Parse all command line arguments into one of these three lists, handling
# boolean and value-based options separately # boolean and value-based options separately
def parse_args(args): def parse_args(args):
@@ -222,7 +377,7 @@ def parse_args(args):
while i < len(args): while i < len(args):
arg = args[i] arg = args[i]
i += 1 i += 1
if not arg.startswith('--'): if not arg.startswith("--"):
unknown_args.append(arg) unknown_args.append(arg)
continue continue
@@ -230,7 +385,7 @@ def parse_args(args):
for option in options: for option in options:
value = None value = None
if option.value: if option.value:
keyval = arg[2:].split('=', 1) keyval = arg[2:].split("=", 1)
key = keyval[0] key = keyval[0]
if option.name != key: if option.name != key:
continue continue
@@ -244,9 +399,9 @@ def parse_args(args):
need_value_args.append(arg) need_value_args.append(arg)
continue continue
else: else:
if arg[2:] == 'enable-' + option.name: if arg[2:] == "enable-" + option.name:
value = True value = True
elif arg[2:] == 'disable-' + option.name: elif arg[2:] == "disable-" + option.name:
value = False value = False
else: else:
continue continue
@@ -263,8 +418,9 @@ def parse_args(args):
# NOTE: here and a few other places, we use [-1] to apply the *last* value # NOTE: here and a few other places, we use [-1] to apply the *last* value
# passed. But if option-checking is enabled, then the known_args loop will # passed. But if option-checking is enabled, then the known_args loop will
# also assert that options are only passed once. # also assert that options are only passed once.
option_checking = ('option-checking' not in known_args option_checking = (
or known_args['option-checking'][-1][1]) "option-checking" not in known_args or known_args["option-checking"][-1][1]
)
if option_checking: if option_checking:
if len(unknown_args) > 0: if len(unknown_args) > 0:
err("Option '" + unknown_args[0] + "' is not recognized") err("Option '" + unknown_args[0] + "' is not recognized")
@@ -272,18 +428,18 @@ def parse_args(args):
err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0])) err("Option '{0}' needs a value ({0}=val)".format(need_value_args[0]))
global VERBOSE global VERBOSE
VERBOSE = 'verbose-configure' in known_args VERBOSE = "verbose-configure" in known_args
config = {} config = {}
set('build.configure-args', args, config) set("build.configure-args", args, config)
apply_args(known_args, option_checking, config) apply_args(known_args, option_checking, config)
return parse_example_config(known_args, config) return parse_example_config(known_args, config)
def build(known_args): def build(known_args):
if 'build' in known_args: if "build" in known_args:
return known_args['build'][-1][1] return known_args["build"][-1][1]
return bootstrap.default_build_triple(verbose=False) return bootstrap.default_build_triple(verbose=False)
@@ -291,7 +447,7 @@ def set(key, value, config):
if isinstance(value, list): if isinstance(value, list):
# Remove empty values, which value.split(',') tends to generate and # Remove empty values, which value.split(',') tends to generate and
# replace single quotes for double quotes to ensure correct parsing. # replace single quotes for double quotes to ensure correct parsing.
value = [v.replace('\'', '"') for v in value if v] value = [v.replace("'", '"') for v in value if v]
s = "{:20} := {}".format(key, value) s = "{:20} := {}".format(key, value)
if len(s) < 70 or VERBOSE: if len(s) < 70 or VERBOSE:
@@ -310,7 +466,7 @@ def set(key, value, config):
for i, part in enumerate(parts): for i, part in enumerate(parts):
if i == len(parts) - 1: if i == len(parts) - 1:
if is_value_list(part) and isinstance(value, str): if is_value_list(part) and isinstance(value, str):
value = value.split(',') value = value.split(",")
arr[part] = value arr[part] = value
else: else:
if part not in arr: if part not in arr:
@@ -321,9 +477,9 @@ def set(key, value, config):
def apply_args(known_args, option_checking, config): def apply_args(known_args, option_checking, config):
for key in known_args: for key in known_args:
# The `set` option is special and can be passed a bunch of times # The `set` option is special and can be passed a bunch of times
if key == 'set': if key == "set":
for _option, value in known_args[key]: for _option, value in known_args[key]:
keyval = value.split('=', 1) keyval = value.split("=", 1)
if len(keyval) == 1 or keyval[1] == "true": if len(keyval) == 1 or keyval[1] == "true":
value = True value = True
elif keyval[1] == "false": elif keyval[1] == "false":
@@ -348,50 +504,55 @@ def apply_args(known_args, option_checking, config):
# that here. # that here.
build_triple = build(known_args) build_triple = build(known_args)
if option.name == 'sccache': if option.name == "sccache":
set('llvm.ccache', 'sccache', config) set("llvm.ccache", "sccache", config)
elif option.name == 'local-rust': elif option.name == "local-rust":
for path in os.environ['PATH'].split(os.pathsep): for path in os.environ["PATH"].split(os.pathsep):
if os.path.exists(path + '/rustc'): if os.path.exists(path + "/rustc"):
set('build.rustc', path + '/rustc', config) set("build.rustc", path + "/rustc", config)
break break
for path in os.environ['PATH'].split(os.pathsep): for path in os.environ["PATH"].split(os.pathsep):
if os.path.exists(path + '/cargo'): if os.path.exists(path + "/cargo"):
set('build.cargo', path + '/cargo', config) set("build.cargo", path + "/cargo", config)
break break
elif option.name == 'local-rust-root': elif option.name == "local-rust-root":
set('build.rustc', value + '/bin/rustc', config) set("build.rustc", value + "/bin/rustc", config)
set('build.cargo', value + '/bin/cargo', config) set("build.cargo", value + "/bin/cargo", config)
elif option.name == 'llvm-root': elif option.name == "llvm-root":
set('target.{}.llvm-config'.format(build_triple), value + '/bin/llvm-config', config) set(
elif option.name == 'llvm-config': "target.{}.llvm-config".format(build_triple),
set('target.{}.llvm-config'.format(build_triple), value, config) value + "/bin/llvm-config",
elif option.name == 'llvm-filecheck': config,
set('target.{}.llvm-filecheck'.format(build_triple), value, config) )
elif option.name == 'tools': elif option.name == "llvm-config":
set('build.tools', value.split(','), config) set("target.{}.llvm-config".format(build_triple), value, config)
elif option.name == 'bootstrap-cache-path': elif option.name == "llvm-filecheck":
set('build.bootstrap-cache-path', value, config) set("target.{}.llvm-filecheck".format(build_triple), value, config)
elif option.name == 'codegen-backends': elif option.name == "tools":
set('rust.codegen-backends', value.split(','), config) set("build.tools", value.split(","), config)
elif option.name == 'host': elif option.name == "bootstrap-cache-path":
set('build.host', value.split(','), config) set("build.bootstrap-cache-path", value, config)
elif option.name == 'target': elif option.name == "codegen-backends":
set('build.target', value.split(','), config) set("rust.codegen-backends", value.split(","), config)
elif option.name == 'full-tools': elif option.name == "host":
set('rust.codegen-backends', ['llvm'], config) set("build.host", value.split(","), config)
set('rust.lld', True, config) elif option.name == "target":
set('rust.llvm-tools', True, config) set("build.target", value.split(","), config)
set('rust.llvm-bitcode-linker', True, config) elif option.name == "full-tools":
set('build.extended', True, config) set("rust.codegen-backends", ["llvm"], config)
elif option.name in ['option-checking', 'verbose-configure']: set("rust.lld", True, config)
set("rust.llvm-tools", True, config)
set("rust.llvm-bitcode-linker", True, config)
set("build.extended", True, config)
elif option.name in ["option-checking", "verbose-configure"]:
# this was handled above # this was handled above
pass pass
elif option.name == 'dist-compression-formats': elif option.name == "dist-compression-formats":
set('dist.compression-formats', value.split(','), config) set("dist.compression-formats", value.split(","), config)
else: else:
raise RuntimeError("unhandled option {}".format(option.name)) raise RuntimeError("unhandled option {}".format(option.name))
# "Parse" the `config.example.toml` file into the various sections, and we'll # "Parse" the `config.example.toml` file into the various sections, and we'll
# use this as a template of a `config.toml` to write out which preserves # use this as a template of a `config.toml` to write out which preserves
# all the various comments and whatnot. # all the various comments and whatnot.
@@ -406,20 +567,22 @@ def parse_example_config(known_args, config):
targets = {} targets = {}
top_level_keys = [] top_level_keys = []
with open(rust_dir + '/config.example.toml') as example_config: with open(rust_dir + "/config.example.toml") as example_config:
example_lines = example_config.read().split("\n") example_lines = example_config.read().split("\n")
for line in example_lines: for line in example_lines:
if cur_section is None: if cur_section is None:
if line.count('=') == 1: if line.count("=") == 1:
top_level_key = line.split('=')[0] top_level_key = line.split("=")[0]
top_level_key = top_level_key.strip(' #') top_level_key = top_level_key.strip(" #")
top_level_keys.append(top_level_key) top_level_keys.append(top_level_key)
if line.startswith('['): if line.startswith("["):
cur_section = line[1:-1] cur_section = line[1:-1]
if cur_section.startswith('target'): if cur_section.startswith("target"):
cur_section = 'target' cur_section = "target"
elif '.' in cur_section: elif "." in cur_section:
raise RuntimeError("don't know how to deal with section: {}".format(cur_section)) raise RuntimeError(
"don't know how to deal with section: {}".format(cur_section)
)
sections[cur_section] = [line] sections[cur_section] = [line]
section_order.append(cur_section) section_order.append(cur_section)
else: else:
@@ -428,22 +591,25 @@ def parse_example_config(known_args, config):
# Fill out the `targets` array by giving all configured targets a copy of the # Fill out the `targets` array by giving all configured targets a copy of the
# `target` section we just loaded from the example config # `target` section we just loaded from the example config
configured_targets = [build(known_args)] configured_targets = [build(known_args)]
if 'build' in config: if "build" in config:
if 'host' in config['build']: if "host" in config["build"]:
configured_targets += config['build']['host'] configured_targets += config["build"]["host"]
if 'target' in config['build']: if "target" in config["build"]:
configured_targets += config['build']['target'] configured_targets += config["build"]["target"]
if 'target' in config: if "target" in config:
for target in config['target']: for target in config["target"]:
configured_targets.append(target) configured_targets.append(target)
for target in configured_targets: for target in configured_targets:
targets[target] = sections['target'][:] targets[target] = sections["target"][:]
# For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target. # For `.` to be valid TOML, it needs to be quoted. But `bootstrap.py` doesn't use a proper TOML parser and fails to parse the target.
# Avoid using quotes unless it's necessary. # Avoid using quotes unless it's necessary.
targets[target][0] = targets[target][0].replace("x86_64-unknown-linux-gnu", "'{}'".format(target) if "." in target else target) targets[target][0] = targets[target][0].replace(
"x86_64-unknown-linux-gnu",
"'{}'".format(target) if "." in target else target,
)
if 'profile' not in config: if "profile" not in config:
set('profile', 'dist', config) set("profile", "dist", config)
configure_file(sections, top_level_keys, targets, config) configure_file(sections, top_level_keys, targets, config)
return section_order, sections, targets return section_order, sections, targets
@@ -467,7 +633,7 @@ def to_toml(value):
else: else:
return "false" return "false"
elif isinstance(value, list): elif isinstance(value, list):
return '[' + ', '.join(map(to_toml, value)) + ']' return "[" + ", ".join(map(to_toml, value)) + "]"
elif isinstance(value, str): elif isinstance(value, str):
# Don't put quotes around numeric values # Don't put quotes around numeric values
if is_number(value): if is_number(value):
@@ -475,9 +641,18 @@ def to_toml(value):
else: else:
return "'" + value + "'" return "'" + value + "'"
elif isinstance(value, dict): elif isinstance(value, dict):
return "{" + ", ".join(map(lambda a: "{} = {}".format(to_toml(a[0]), to_toml(a[1])), value.items())) + "}" return (
"{"
+ ", ".join(
map(
lambda a: "{} = {}".format(to_toml(a[0]), to_toml(a[1])),
value.items(),
)
)
+ "}"
)
else: else:
raise RuntimeError('no toml') raise RuntimeError("no toml")
def configure_section(lines, config): def configure_section(lines, config):
@@ -485,7 +660,7 @@ def configure_section(lines, config):
value = config[key] value = config[key]
found = False found = False
for i, line in enumerate(lines): for i, line in enumerate(lines):
if not line.startswith('#' + key + ' = '): if not line.startswith("#" + key + " = "):
continue continue
found = True found = True
lines[i] = "{} = {}".format(key, to_toml(value)) lines[i] = "{} = {}".format(key, to_toml(value))
@@ -501,7 +676,9 @@ def configure_section(lines, config):
def configure_top_level_key(lines, top_level_key, value): def configure_top_level_key(lines, top_level_key, value):
for i, line in enumerate(lines): for i, line in enumerate(lines):
if line.startswith('#' + top_level_key + ' = ') or line.startswith(top_level_key + ' = '): if line.startswith("#" + top_level_key + " = ") or line.startswith(
top_level_key + " = "
):
lines[i] = "{} = {}".format(top_level_key, to_toml(value)) lines[i] = "{} = {}".format(top_level_key, to_toml(value))
return return
@@ -512,11 +689,13 @@ def configure_top_level_key(lines, top_level_key, value):
def configure_file(sections, top_level_keys, targets, config): def configure_file(sections, top_level_keys, targets, config):
for section_key, section_config in config.items(): for section_key, section_config in config.items():
if section_key not in sections and section_key not in top_level_keys: if section_key not in sections and section_key not in top_level_keys:
raise RuntimeError("config key {} not in sections or top_level_keys".format(section_key)) raise RuntimeError(
"config key {} not in sections or top_level_keys".format(section_key)
)
if section_key in top_level_keys: if section_key in top_level_keys:
configure_top_level_key(sections[None], section_key, section_config) configure_top_level_key(sections[None], section_key, section_config)
elif section_key == 'target': elif section_key == "target":
for target in section_config: for target in section_config:
configure_section(targets[target], section_config[target]) configure_section(targets[target], section_config[target])
else: else:
@@ -536,18 +715,19 @@ def write_uncommented(target, f):
block = [] block = []
is_comment = True is_comment = True
continue continue
is_comment = is_comment and line.startswith('#') is_comment = is_comment and line.startswith("#")
return f return f
def write_config_toml(writer, section_order, targets, sections): def write_config_toml(writer, section_order, targets, sections):
for section in section_order: for section in section_order:
if section == 'target': if section == "target":
for target in targets: for target in targets:
writer = write_uncommented(targets[target], writer) writer = write_uncommented(targets[target], writer)
else: else:
writer = write_uncommented(sections[section], writer) writer = write_uncommented(sections[section], writer)
def quit_if_file_exists(file): def quit_if_file_exists(file):
if os.path.isfile(file): if os.path.isfile(file):
msg = "Existing '{}' detected. Exiting".format(file) msg = "Existing '{}' detected. Exiting".format(file)
@@ -559,9 +739,10 @@ def quit_if_file_exists(file):
err(msg) err(msg)
if __name__ == "__main__": if __name__ == "__main__":
# If 'config.toml' already exists, exit the script at this point # If 'config.toml' already exists, exit the script at this point
quit_if_file_exists('config.toml') quit_if_file_exists("config.toml")
if "GITHUB_ACTIONS" in os.environ: if "GITHUB_ACTIONS" in os.environ:
print("::group::Configure the build") print("::group::Configure the build")
@@ -575,13 +756,13 @@ if __name__ == "__main__":
# order that we read it in. # order that we read it in.
p("") p("")
p("writing `config.toml` in current directory") p("writing `config.toml` in current directory")
with bootstrap.output('config.toml') as f: with bootstrap.output("config.toml") as f:
write_config_toml(f, section_order, targets, sections) write_config_toml(f, section_order, targets, sections)
with bootstrap.output('Makefile') as f: with bootstrap.output("Makefile") as f:
contents = os.path.join(rust_dir, 'src', 'bootstrap', 'mk', 'Makefile.in') contents = os.path.join(rust_dir, "src", "bootstrap", "mk", "Makefile.in")
contents = open(contents).read() contents = open(contents).read()
contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + '/') contents = contents.replace("$(CFG_SRC_DIR)", rust_dir + "/")
contents = contents.replace("$(CFG_PYTHON)", sys.executable) contents = contents.replace("$(CFG_PYTHON)", sys.executable)
f.write(contents) f.write(contents)
src/ci/cpu-usage-over-time.py
+25 -11
View File
@@ -40,12 +40,13 @@ import time
# Python 3.3 changed the value of `sys.platform` on Linux from "linux2" to just # Python 3.3 changed the value of `sys.platform` on Linux from "linux2" to just
# "linux". We check here with `.startswith` to keep compatibility with older # "linux". We check here with `.startswith` to keep compatibility with older
# Python versions (especially Python 2.7). # Python versions (especially Python 2.7).
if sys.platform.startswith('linux'): if sys.platform.startswith("linux"):
class State: class State:
def __init__(self): def __init__(self):
with open('/proc/stat', 'r') as file: with open("/proc/stat", "r") as file:
data = file.readline().split() data = file.readline().split()
if data[0] != 'cpu': if data[0] != "cpu":
raise Exception('did not start with "cpu"') raise Exception('did not start with "cpu"')
self.user = int(data[1]) self.user = int(data[1])
self.nice = int(data[2]) self.nice = int(data[2])
@@ -69,10 +70,21 @@ if sys.platform.startswith('linux'):
steal = self.steal - prev.steal steal = self.steal - prev.steal
guest = self.guest - prev.guest guest = self.guest - prev.guest
guest_nice = self.guest_nice - prev.guest_nice guest_nice = self.guest_nice - prev.guest_nice
total = user + nice + system + idle + iowait + irq + softirq + steal + guest + guest_nice total = (
user
+ nice
+ system
+ idle
+ iowait
+ irq
+ softirq
+ steal
+ guest
+ guest_nice
)
return float(idle) / float(total) * 100 return float(idle) / float(total) * 100
elif sys.platform == 'win32': elif sys.platform == "win32":
from ctypes.wintypes import DWORD from ctypes.wintypes import DWORD
from ctypes import Structure, windll, WinError, GetLastError, byref from ctypes import Structure, windll, WinError, GetLastError, byref
@@ -104,9 +116,10 @@ elif sys.platform == 'win32':
kernel = self.kernel - prev.kernel kernel = self.kernel - prev.kernel
return float(idle) / float(user + kernel) * 100 return float(idle) / float(user + kernel) * 100
elif sys.platform == 'darwin': elif sys.platform == "darwin":
from ctypes import * from ctypes import *
libc = cdll.LoadLibrary('/usr/lib/libc.dylib')
libc = cdll.LoadLibrary("/usr/lib/libc.dylib")
class host_cpu_load_info_data_t(Structure): class host_cpu_load_info_data_t(Structure):
_fields_ = [("cpu_ticks", c_uint * 4)] _fields_ = [("cpu_ticks", c_uint * 4)]
@@ -116,7 +129,7 @@ elif sys.platform == 'darwin':
c_uint, c_uint,
c_int, c_int,
POINTER(host_cpu_load_info_data_t), POINTER(host_cpu_load_info_data_t),
POINTER(c_int) POINTER(c_int),
] ]
host_statistics.restype = c_int host_statistics.restype = c_int
@@ -124,13 +137,14 @@ elif sys.platform == 'darwin':
CPU_STATE_SYSTEM = 1 CPU_STATE_SYSTEM = 1
CPU_STATE_IDLE = 2 CPU_STATE_IDLE = 2
CPU_STATE_NICE = 3 CPU_STATE_NICE = 3
class State: class State:
def __init__(self): def __init__(self):
stats = host_cpu_load_info_data_t() stats = host_cpu_load_info_data_t()
count = c_int(4) # HOST_CPU_LOAD_INFO_COUNT count = c_int(4) # HOST_CPU_LOAD_INFO_COUNT
err = libc.host_statistics( err = libc.host_statistics(
libc.mach_host_self(), libc.mach_host_self(),
c_int(3), # HOST_CPU_LOAD_INFO c_int(3), # HOST_CPU_LOAD_INFO
byref(stats), byref(stats),
byref(count), byref(count),
) )
@@ -148,7 +162,7 @@ elif sys.platform == 'darwin':
return float(idle) / float(user + system + idle + nice) * 100.0 return float(idle) / float(user + system + idle + nice) * 100.0
else: else:
print('unknown platform', sys.platform) print("unknown platform", sys.platform)
sys.exit(1) sys.exit(1)
cur_state = State() cur_state = State()
src/ci/docker/host-x86_64/test-various/uefi_qemu_test/run.py
+75 -72
View File
@@ -8,78 +8,79 @@ import tempfile
from pathlib import Path from pathlib import Path
TARGET_AARCH64 = 'aarch64-unknown-uefi' TARGET_AARCH64 = "aarch64-unknown-uefi"
TARGET_I686 = 'i686-unknown-uefi' TARGET_I686 = "i686-unknown-uefi"
TARGET_X86_64 = 'x86_64-unknown-uefi' TARGET_X86_64 = "x86_64-unknown-uefi"
def run(*cmd, capture=False, check=True, env=None, timeout=None): def run(*cmd, capture=False, check=True, env=None, timeout=None):
"""Print and run a command, optionally capturing the output.""" """Print and run a command, optionally capturing the output."""
cmd = [str(p) for p in cmd] cmd = [str(p) for p in cmd]
print(' '.join(cmd)) print(" ".join(cmd))
return subprocess.run(cmd, return subprocess.run(
capture_output=capture, cmd, capture_output=capture, check=check, env=env, text=True, timeout=timeout
check=check, )
env=env,
text=True,
timeout=timeout)
def build_and_run(tmp_dir, target): def build_and_run(tmp_dir, target):
if target == TARGET_AARCH64: if target == TARGET_AARCH64:
boot_file_name = 'bootaa64.efi' boot_file_name = "bootaa64.efi"
ovmf_dir = Path('/usr/share/AAVMF') ovmf_dir = Path("/usr/share/AAVMF")
ovmf_code = 'AAVMF_CODE.fd' ovmf_code = "AAVMF_CODE.fd"
ovmf_vars = 'AAVMF_VARS.fd' ovmf_vars = "AAVMF_VARS.fd"
qemu = 'qemu-system-aarch64' qemu = "qemu-system-aarch64"
machine = 'virt' machine = "virt"
cpu = 'cortex-a72' cpu = "cortex-a72"
elif target == TARGET_I686: elif target == TARGET_I686:
boot_file_name = 'bootia32.efi' boot_file_name = "bootia32.efi"
ovmf_dir = Path('/usr/share/OVMF') ovmf_dir = Path("/usr/share/OVMF")
ovmf_code = 'OVMF32_CODE_4M.secboot.fd' ovmf_code = "OVMF32_CODE_4M.secboot.fd"
ovmf_vars = 'OVMF32_VARS_4M.fd' ovmf_vars = "OVMF32_VARS_4M.fd"
# The i686 target intentionally uses 64-bit qemu; the important # The i686 target intentionally uses 64-bit qemu; the important
# difference is that the OVMF code provides a 32-bit environment. # difference is that the OVMF code provides a 32-bit environment.
qemu = 'qemu-system-x86_64' qemu = "qemu-system-x86_64"
machine = 'q35' machine = "q35"
cpu = 'qemu64' cpu = "qemu64"
elif target == TARGET_X86_64: elif target == TARGET_X86_64:
boot_file_name = 'bootx64.efi' boot_file_name = "bootx64.efi"
ovmf_dir = Path('/usr/share/OVMF') ovmf_dir = Path("/usr/share/OVMF")
ovmf_code = 'OVMF_CODE.fd' ovmf_code = "OVMF_CODE.fd"
ovmf_vars = 'OVMF_VARS.fd' ovmf_vars = "OVMF_VARS.fd"
qemu = 'qemu-system-x86_64' qemu = "qemu-system-x86_64"
machine = 'q35' machine = "q35"
cpu = 'qemu64' cpu = "qemu64"
else: else:
raise KeyError('invalid target') raise KeyError("invalid target")
host_artifacts = Path('/checkout/obj/build/x86_64-unknown-linux-gnu') host_artifacts = Path("/checkout/obj/build/x86_64-unknown-linux-gnu")
stage0 = host_artifacts / 'stage0/bin' stage0 = host_artifacts / "stage0/bin"
stage2 = host_artifacts / 'stage2/bin' stage2 = host_artifacts / "stage2/bin"
env = dict(os.environ) env = dict(os.environ)
env['PATH'] = '{}:{}:{}'.format(stage2, stage0, env['PATH']) env["PATH"] = "{}:{}:{}".format(stage2, stage0, env["PATH"])
# Copy the test create into `tmp_dir`. # Copy the test create into `tmp_dir`.
test_crate = Path(tmp_dir) / 'uefi_qemu_test' test_crate = Path(tmp_dir) / "uefi_qemu_test"
shutil.copytree('/uefi_qemu_test', test_crate) shutil.copytree("/uefi_qemu_test", test_crate)
# Build the UEFI executable. # Build the UEFI executable.
run('cargo', run(
'build', "cargo",
'--manifest-path', "build",
test_crate / 'Cargo.toml', "--manifest-path",
'--target', test_crate / "Cargo.toml",
"--target",
target, target,
env=env) env=env,
)
# Create a mock EFI System Partition in a subdirectory. # Create a mock EFI System Partition in a subdirectory.
esp = test_crate / 'esp' esp = test_crate / "esp"
boot = esp / 'efi/boot' boot = esp / "efi/boot"
os.makedirs(boot, exist_ok=True) os.makedirs(boot, exist_ok=True)
# Copy the executable into the ESP. # Copy the executable into the ESP.
src_exe_path = test_crate / 'target' / target / 'debug/uefi_qemu_test.efi' src_exe_path = test_crate / "target" / target / "debug/uefi_qemu_test.efi"
shutil.copy(src_exe_path, boot / boot_file_name) shutil.copy(src_exe_path, boot / boot_file_name)
print(src_exe_path, boot / boot_file_name) print(src_exe_path, boot / boot_file_name)
@@ -89,37 +90,39 @@ def build_and_run(tmp_dir, target):
# Make a writable copy of the vars file. aarch64 doesn't boot # Make a writable copy of the vars file. aarch64 doesn't boot
# correctly with read-only vars. # correctly with read-only vars.
ovmf_rw_vars = Path(tmp_dir) / 'vars.fd' ovmf_rw_vars = Path(tmp_dir) / "vars.fd"
shutil.copy(ovmf_vars, ovmf_rw_vars) shutil.copy(ovmf_vars, ovmf_rw_vars)
# Run the executable in QEMU and capture the output. # Run the executable in QEMU and capture the output.
output = run(qemu, output = run(
'-machine', qemu,
machine, "-machine",
'-cpu', machine,
cpu, "-cpu",
'-display', cpu,
'none', "-display",
'-serial', "none",
'stdio', "-serial",
'-drive', "stdio",
f'if=pflash,format=raw,readonly=on,file={ovmf_code}', "-drive",
'-drive', f"if=pflash,format=raw,readonly=on,file={ovmf_code}",
f'if=pflash,format=raw,readonly=off,file={ovmf_rw_vars}', "-drive",
'-drive', f"if=pflash,format=raw,readonly=off,file={ovmf_rw_vars}",
f'format=raw,file=fat:rw:{esp}', "-drive",
capture=True, f"format=raw,file=fat:rw:{esp}",
check=True, capture=True,
# Set a timeout to kill the VM in case something goes wrong. check=True,
timeout=60).stdout # Set a timeout to kill the VM in case something goes wrong.
timeout=60,
).stdout
if 'Hello World!' in output: if "Hello World!" in output:
print('VM produced expected output') print("VM produced expected output")
else: else:
print('unexpected VM output:') print("unexpected VM output:")
print('---start---') print("---start---")
print(output) print(output)
print('---end---') print("---end---")
sys.exit(1) sys.exit(1)
src/ci/docker/scripts/android-sdk-manager.py
+47 -13
View File
@@ -35,6 +35,7 @@ MIRROR_BUCKET = "rust-lang-ci-mirrors"
MIRROR_BUCKET_REGION = "us-west-1" MIRROR_BUCKET_REGION = "us-west-1"
MIRROR_BASE_DIR = "rustc/android/" MIRROR_BASE_DIR = "rustc/android/"
class Package: class Package:
def __init__(self, path, url, sha1, deps=None): def __init__(self, path, url, sha1, deps=None):
if deps is None: if deps is None:
@@ -53,18 +54,25 @@ class Package:
sha1 = hashlib.sha1(f.read()).hexdigest() sha1 = hashlib.sha1(f.read()).hexdigest()
if sha1 != self.sha1: if sha1 != self.sha1:
raise RuntimeError( raise RuntimeError(
"hash mismatch for package " + self.path + ": " + "hash mismatch for package "
sha1 + " vs " + self.sha1 + " (known good)" + self.path
+ ": "
+ sha1
+ " vs "
+ self.sha1
+ " (known good)"
) )
return file return file
def __repr__(self): def __repr__(self):
return "<Package "+self.path+" at "+self.url+" (sha1="+self.sha1+")" return "<Package " + self.path + " at " + self.url + " (sha1=" + self.sha1 + ")"
def fetch_url(url): def fetch_url(url):
page = urllib.request.urlopen(url) page = urllib.request.urlopen(url)
return page.read() return page.read()
def fetch_repository(base, repo_url): def fetch_repository(base, repo_url):
packages = {} packages = {}
root = ET.fromstring(fetch_url(base + repo_url)) root = ET.fromstring(fetch_url(base + repo_url))
@@ -92,12 +100,14 @@ def fetch_repository(base, repo_url):
return packages return packages
def fetch_repositories(): def fetch_repositories():
packages = {} packages = {}
for repo in REPOSITORIES: for repo in REPOSITORIES:
packages.update(fetch_repository(BASE_REPOSITORY, repo)) packages.update(fetch_repository(BASE_REPOSITORY, repo))
return packages return packages
class Lockfile: class Lockfile:
def __init__(self, path): def __init__(self, path):
self.path = path self.path = path
@@ -123,6 +133,7 @@ class Lockfile:
for package in packages: for package in packages:
f.write(package.path + " " + package.url + " " + package.sha1 + "\n") f.write(package.path + " " + package.url + " " + package.sha1 + "\n")
def cli_add_to_lockfile(args): def cli_add_to_lockfile(args):
lockfile = Lockfile(args.lockfile) lockfile = Lockfile(args.lockfile)
packages = fetch_repositories() packages = fetch_repositories()
@@ -130,28 +141,49 @@ def cli_add_to_lockfile(args):
lockfile.add(packages, package) lockfile.add(packages, package)
lockfile.save() lockfile.save()
def cli_update_mirror(args): def cli_update_mirror(args):
lockfile = Lockfile(args.lockfile) lockfile = Lockfile(args.lockfile)
for package in lockfile.packages.values(): for package in lockfile.packages.values():
path = package.download(BASE_REPOSITORY) path = package.download(BASE_REPOSITORY)
subprocess.run([ subprocess.run(
"aws", "s3", "mv", path, [
"s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url, "aws",
"--profile=" + args.awscli_profile, "s3",
], check=True) "mv",
path,
"s3://" + MIRROR_BUCKET + "/" + MIRROR_BASE_DIR + package.url,
"--profile=" + args.awscli_profile,
],
check=True,
)
def cli_install(args): def cli_install(args):
lockfile = Lockfile(args.lockfile) lockfile = Lockfile(args.lockfile)
for package in lockfile.packages.values(): for package in lockfile.packages.values():
# Download the file from the mirror into a temp file # Download the file from the mirror into a temp file
url = "https://" + MIRROR_BUCKET + ".s3-" + MIRROR_BUCKET_REGION + \ url = (
".amazonaws.com/" + MIRROR_BASE_DIR "https://"
+ MIRROR_BUCKET
+ ".s3-"
+ MIRROR_BUCKET_REGION
+ ".amazonaws.com/"
+ MIRROR_BASE_DIR
)
downloaded = package.download(url) downloaded = package.download(url)
# Extract the file in a temporary directory # Extract the file in a temporary directory
extract_dir = tempfile.mkdtemp() extract_dir = tempfile.mkdtemp()
subprocess.run([ subprocess.run(
"unzip", "-q", downloaded, "-d", extract_dir, [
], check=True) "unzip",
"-q",
downloaded,
"-d",
extract_dir,
],
check=True,
)
# Figure out the prefix used in the zip # Figure out the prefix used in the zip
subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")] subdirs = [d for d in os.listdir(extract_dir) if not d.startswith(".")]
if len(subdirs) != 1: if len(subdirs) != 1:
@@ -162,6 +194,7 @@ def cli_install(args):
os.rename(os.path.join(extract_dir, subdirs[0]), dest) os.rename(os.path.join(extract_dir, subdirs[0]), dest)
os.unlink(downloaded) os.unlink(downloaded)
def cli(): def cli():
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers() subparsers = parser.add_subparsers()
@@ -187,5 +220,6 @@ def cli():
exit(1) exit(1)
args.func(args) args.func(args)
if __name__ == "__main__": if __name__ == "__main__":
cli() cli()
src/ci/docker/scripts/fuchsia-test-runner.py
+3 -5
View File
@@ -588,7 +588,7 @@ class TestEnvironment:
"--repo-path", "--repo-path",
self.repo_dir(), self.repo_dir(),
"--repository", "--repository",
self.TEST_REPO_NAME self.TEST_REPO_NAME,
], ],
env=ffx_env, env=ffx_env,
stdout_handler=self.subprocess_logger.debug, stdout_handler=self.subprocess_logger.debug,
@@ -619,9 +619,7 @@ class TestEnvironment:
# `facet` statement required for TCP testing via # `facet` statement required for TCP testing via
# protocol `fuchsia.posix.socket.Provider`. See # protocol `fuchsia.posix.socket.Provider`. See
# https://fuchsia.dev/fuchsia-src/development/testing/components/test_runner_framework?hl=en#legacy_non-hermetic_tests # https://fuchsia.dev/fuchsia-src/development/testing/components/test_runner_framework?hl=en#legacy_non-hermetic_tests
CML_TEMPLATE: ClassVar[ CML_TEMPLATE: ClassVar[str] = """
str
] = """
{{ {{
program: {{ program: {{
runner: "elf_test_runner", runner: "elf_test_runner",
@@ -994,7 +992,7 @@ class TestEnvironment:
"repository", "repository",
"server", "server",
"stop", "stop",
self.TEST_REPO_NAME self.TEST_REPO_NAME,
], ],
env=self.ffx_cmd_env(), env=self.ffx_cmd_env(),
stdout_handler=self.subprocess_logger.debug, stdout_handler=self.subprocess_logger.debug,
src/ci/github-actions/calculate-job-matrix.py
+9 -4
View File
@@ -7,6 +7,7 @@ be executed on CI.
It reads job definitions from `src/ci/github-actions/jobs.yml` It reads job definitions from `src/ci/github-actions/jobs.yml`
and filters them based on the event that happened on CI. and filters them based on the event that happened on CI.
""" """
import dataclasses import dataclasses
import json import json
import logging import logging
@@ -94,7 +95,7 @@ def find_run_type(ctx: GitHubCtx) -> Optional[WorkflowRunType]:
try_build = ctx.ref in ( try_build = ctx.ref in (
"refs/heads/try", "refs/heads/try",
"refs/heads/try-perf", "refs/heads/try-perf",
"refs/heads/automation/bors/try" "refs/heads/automation/bors/try",
) )
# Unrolled branch from a rollup for testing perf # Unrolled branch from a rollup for testing perf
@@ -135,11 +136,15 @@ def calculate_jobs(run_type: WorkflowRunType, job_data: Dict[str, Any]) -> List[
continue continue
jobs.append(job[0]) jobs.append(job[0])
if unknown_jobs: if unknown_jobs:
raise Exception(f"Custom job(s) `{unknown_jobs}` not found in auto jobs") raise Exception(
f"Custom job(s) `{unknown_jobs}` not found in auto jobs"
)
return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"]) return add_base_env(name_jobs(jobs, "try"), job_data["envs"]["try"])
elif isinstance(run_type, AutoRunType): elif isinstance(run_type, AutoRunType):
return add_base_env(name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"]) return add_base_env(
name_jobs(job_data["auto"], "auto"), job_data["envs"]["auto"]
)
return [] return []
@@ -161,7 +166,7 @@ def get_github_ctx() -> GitHubCtx:
event_name=event_name, event_name=event_name,
ref=os.environ["GITHUB_REF"], ref=os.environ["GITHUB_REF"],
repository=os.environ["GITHUB_REPOSITORY"], repository=os.environ["GITHUB_REPOSITORY"],
commit_message=commit_message commit_message=commit_message,
) )
src/ci/scripts/upload-build-metrics.py
+9 -12
View File
@@ -19,6 +19,7 @@ $ python3 upload-build-metrics.py <path-to-CPU-usage-CSV>
`path-to-CPU-usage-CSV` is a path to a CSV generated by the `src/ci/cpu-usage-over-time.py` script. `path-to-CPU-usage-CSV` is a path to a CSV generated by the `src/ci/cpu-usage-over-time.py` script.
""" """
import argparse import argparse
import csv import csv
import os import os
@@ -31,7 +32,7 @@ from typing import List
def load_cpu_usage(path: Path) -> List[float]: def load_cpu_usage(path: Path) -> List[float]:
usage = [] usage = []
with open(path) as f: with open(path) as f:
reader = csv.reader(f, delimiter=',') reader = csv.reader(f, delimiter=",")
for row in reader: for row in reader:
# The log might contain incomplete rows or some Python exception # The log might contain incomplete rows or some Python exception
if len(row) == 2: if len(row) == 2:
@@ -50,25 +51,21 @@ def upload_datadog_measure(name: str, value: float):
print(f"Metric {name}: {value:.4f}") print(f"Metric {name}: {value:.4f}")
datadog_cmd = "datadog-ci" datadog_cmd = "datadog-ci"
if os.getenv("GITHUB_ACTIONS") is not None and sys.platform.lower().startswith("win"): if os.getenv("GITHUB_ACTIONS") is not None and sys.platform.lower().startswith(
"win"
):
# Due to weird interaction of MSYS2 and Python, we need to use an absolute path, # Due to weird interaction of MSYS2 and Python, we need to use an absolute path,
# and also specify the ".cmd" at the end. See https://github.com/rust-lang/rust/pull/125771. # and also specify the ".cmd" at the end. See https://github.com/rust-lang/rust/pull/125771.
datadog_cmd = "C:\\npm\\prefix\\datadog-ci.cmd" datadog_cmd = "C:\\npm\\prefix\\datadog-ci.cmd"
subprocess.run([ subprocess.run(
datadog_cmd, [datadog_cmd, "measure", "--level", "job", "--measures", f"{name}:{value}"],
"measure", check=False,
"--level", "job",
"--measures", f"{name}:{value}"
],
check=False
) )
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(prog="DataDog metric uploader")
prog="DataDog metric uploader"
)
parser.add_argument("cpu-usage-history-csv") parser.add_argument("cpu-usage-history-csv")
args = parser.parse_args() args = parser.parse_args()
src/etc/dec2flt_table.py
+24 -22
View File
@@ -13,6 +13,7 @@ i.e., within 0.5 ULP of the true value.
Adapted from Daniel Lemire's fast_float ``table_generation.py``, Adapted from Daniel Lemire's fast_float ``table_generation.py``,
available here: <https://github.com/fastfloat/fast_float/blob/main/script/table_generation.py>. available here: <https://github.com/fastfloat/fast_float/blob/main/script/table_generation.py>.
""" """
from __future__ import print_function from __future__ import print_function
from math import ceil, floor, log from math import ceil, floor, log
from collections import deque from collections import deque
@@ -34,6 +35,7 @@ STATIC_WARNING = """
// the final binary. // the final binary.
""" """
def main(): def main():
min_exp = minimum_exponent(10) min_exp = minimum_exponent(10)
max_exp = maximum_exponent(10) max_exp = maximum_exponent(10)
@@ -41,10 +43,10 @@ def main():
print(HEADER.strip()) print(HEADER.strip())
print() print()
print('pub const SMALLEST_POWER_OF_FIVE: i32 = {};'.format(min_exp)) print("pub const SMALLEST_POWER_OF_FIVE: i32 = {};".format(min_exp))
print('pub const LARGEST_POWER_OF_FIVE: i32 = {};'.format(max_exp)) print("pub const LARGEST_POWER_OF_FIVE: i32 = {};".format(max_exp))
print('pub const N_POWERS_OF_FIVE: usize = ', end='') print("pub const N_POWERS_OF_FIVE: usize = ", end="")
print('(LARGEST_POWER_OF_FIVE - SMALLEST_POWER_OF_FIVE + 1) as usize;') print("(LARGEST_POWER_OF_FIVE - SMALLEST_POWER_OF_FIVE + 1) as usize;")
print() print()
print_proper_powers(min_exp, max_exp, bias) print_proper_powers(min_exp, max_exp, bias)
@@ -54,7 +56,7 @@ def minimum_exponent(base):
def maximum_exponent(base): def maximum_exponent(base):
return floor(log(1.7976931348623157e+308, base)) return floor(log(1.7976931348623157e308, base))
def print_proper_powers(min_exp, max_exp, bias): def print_proper_powers(min_exp, max_exp, bias):
@@ -64,46 +66,46 @@ def print_proper_powers(min_exp, max_exp, bias):
# 2^(2b)/(5^−q) with b=64 + int(math.ceil(log2(5^−q))) # 2^(2b)/(5^−q) with b=64 + int(math.ceil(log2(5^−q)))
powers = [] powers = []
for q in range(min_exp, 0): for q in range(min_exp, 0):
power5 = 5 ** -q power5 = 5**-q
z = 0 z = 0
while (1 << z) < power5: while (1 << z) < power5:
z += 1 z += 1
if q >= -27: if q >= -27:
b = z + 127 b = z + 127
c = 2 ** b // power5 + 1 c = 2**b // power5 + 1
powers.append((c, q)) powers.append((c, q))
else: else:
b = 2 * z + 2 * 64 b = 2 * z + 2 * 64
c = 2 ** b // power5 + 1 c = 2**b // power5 + 1
# truncate # truncate
while c >= (1<<128): while c >= (1 << 128):
c //= 2 c //= 2
powers.append((c, q)) powers.append((c, q))
# Add positive exponents # Add positive exponents
for q in range(0, max_exp + 1): for q in range(0, max_exp + 1):
power5 = 5 ** q power5 = 5**q
# move the most significant bit in position # move the most significant bit in position
while power5 < (1<<127): while power5 < (1 << 127):
power5 *= 2 power5 *= 2
# *truncate* # *truncate*
while power5 >= (1<<128): while power5 >= (1 << 128):
power5 //= 2 power5 //= 2
powers.append((power5, q)) powers.append((power5, q))
# Print the powers. # Print the powers.
print(STATIC_WARNING.strip()) print(STATIC_WARNING.strip())
print('#[rustfmt::skip]') print("#[rustfmt::skip]")
typ = '[(u64, u64); N_POWERS_OF_FIVE]' typ = "[(u64, u64); N_POWERS_OF_FIVE]"
print('pub static POWER_OF_FIVE_128: {} = ['.format(typ)) print("pub static POWER_OF_FIVE_128: {} = [".format(typ))
for c, exp in powers: for c, exp in powers:
hi = '0x{:x}'.format(c // (1 << 64)) hi = "0x{:x}".format(c // (1 << 64))
lo = '0x{:x}'.format(c % (1 << 64)) lo = "0x{:x}".format(c % (1 << 64))
value = ' ({}, {}), '.format(hi, lo) value = " ({}, {}), ".format(hi, lo)
comment = '// {}^{}'.format(5, exp) comment = "// {}^{}".format(5, exp)
print(value.ljust(46, ' ') + comment) print(value.ljust(46, " ") + comment)
print('];') print("];")
if __name__ == '__main__': if __name__ == "__main__":
main() main()
src/etc/gdb_load_rust_pretty_printers.py
+1
View File
@@ -1,6 +1,7 @@
# Add this folder to the python sys path; GDB Python-interpreter will now find modules in this path # Add this folder to the python sys path; GDB Python-interpreter will now find modules in this path
import sys import sys
from os import path from os import path
self_dir = path.dirname(path.realpath(__file__)) self_dir = path.dirname(path.realpath(__file__))
sys.path.append(self_dir) sys.path.append(self_dir)
src/etc/gdb_lookup.py
+5 -2
View File
@@ -6,8 +6,11 @@ from gdb_providers import *
from rust_types import * from rust_types import *
_gdb_version_matched = re.search('([0-9]+)\\.([0-9]+)', gdb.VERSION) _gdb_version_matched = re.search("([0-9]+)\\.([0-9]+)", gdb.VERSION)
gdb_version = [int(num) for num in _gdb_version_matched.groups()] if _gdb_version_matched else [] gdb_version = (
[int(num) for num in _gdb_version_matched.groups()] if _gdb_version_matched else []
)
def register_printers(objfile): def register_printers(objfile):
objfile.pretty_printers.append(printer) objfile.pretty_printers.append(printer)
src/etc/gdb_providers.py
+26 -9
View File
@@ -21,7 +21,7 @@ def unwrap_unique_or_non_null(unique_or_nonnull):
# GDB 14 has a tag class that indicates that extension methods are ok # GDB 14 has a tag class that indicates that extension methods are ok
# to call. Use of this tag only requires that printers hide local # to call. Use of this tag only requires that printers hide local
# attributes and methods by prefixing them with "_". # attributes and methods by prefixing them with "_".
if hasattr(gdb, 'ValuePrinter'): if hasattr(gdb, "ValuePrinter"):
printer_base = gdb.ValuePrinter printer_base = gdb.ValuePrinter
else: else:
printer_base = object printer_base = object
@@ -98,7 +98,7 @@ class StdStrProvider(printer_base):
def _enumerate_array_elements(element_ptrs): def _enumerate_array_elements(element_ptrs):
for (i, element_ptr) in enumerate(element_ptrs): for i, element_ptr in enumerate(element_ptrs):
key = "[{}]".format(i) key = "[{}]".format(i)
element = element_ptr.dereference() element = element_ptr.dereference()
@@ -173,7 +173,8 @@ class StdVecDequeProvider(printer_base):
def children(self): def children(self):
return _enumerate_array_elements( return _enumerate_array_elements(
(self._data_ptr + ((self._head + index) % self._cap)) for index in xrange(self._size) (self._data_ptr + ((self._head + index) % self._cap))
for index in xrange(self._size)
) )
@staticmethod @staticmethod
@@ -270,7 +271,9 @@ def children_of_btree_map(map):
# Yields each key/value pair in the node and in any child nodes. # Yields each key/value pair in the node and in any child nodes.
def children_of_node(node_ptr, height): def children_of_node(node_ptr, height):
def cast_to_internal(node): def cast_to_internal(node):
internal_type_name = node.type.target().name.replace("LeafNode", "InternalNode", 1) internal_type_name = node.type.target().name.replace(
"LeafNode", "InternalNode", 1
)
internal_type = gdb.lookup_type(internal_type_name) internal_type = gdb.lookup_type(internal_type_name)
return node.cast(internal_type.pointer()) return node.cast(internal_type.pointer())
@@ -293,8 +296,16 @@ def children_of_btree_map(map):
# Avoid "Cannot perform pointer math on incomplete type" on zero-sized arrays. # Avoid "Cannot perform pointer math on incomplete type" on zero-sized arrays.
key_type_size = keys.type.sizeof key_type_size = keys.type.sizeof
val_type_size = vals.type.sizeof val_type_size = vals.type.sizeof
key = keys[i]["value"]["value"] if key_type_size > 0 else gdb.parse_and_eval("()") key = (
val = vals[i]["value"]["value"] if val_type_size > 0 else gdb.parse_and_eval("()") keys[i]["value"]["value"]
if key_type_size > 0
else gdb.parse_and_eval("()")
)
val = (
vals[i]["value"]["value"]
if val_type_size > 0
else gdb.parse_and_eval("()")
)
yield key, val yield key, val
if map["length"] > 0: if map["length"] > 0:
@@ -352,7 +363,7 @@ class StdOldHashMapProvider(printer_base):
self._hashes = self._table["hashes"] self._hashes = self._table["hashes"]
self._hash_uint_type = self._hashes.type self._hash_uint_type = self._hashes.type
self._hash_uint_size = self._hashes.type.sizeof self._hash_uint_size = self._hashes.type.sizeof
self._modulo = 2 ** self._hash_uint_size self._modulo = 2**self._hash_uint_size
self._data_ptr = self._hashes[ZERO_FIELD]["pointer"] self._data_ptr = self._hashes[ZERO_FIELD]["pointer"]
self._capacity_mask = int(self._table["capacity_mask"]) self._capacity_mask = int(self._table["capacity_mask"])
@@ -382,8 +393,14 @@ class StdOldHashMapProvider(printer_base):
hashes = self._hash_uint_size * self._capacity hashes = self._hash_uint_size * self._capacity
align = self._pair_type_size align = self._pair_type_size
len_rounded_up = (((((hashes + align) % self._modulo - 1) % self._modulo) & ~( len_rounded_up = (
(align - 1) % self._modulo)) % self._modulo - hashes) % self._modulo (
(((hashes + align) % self._modulo - 1) % self._modulo)
& ~((align - 1) % self._modulo)
)
% self._modulo
- hashes
) % self._modulo
pairs_offset = hashes + len_rounded_up pairs_offset = hashes + len_rounded_up
pairs_start = gdb.Value(start + pairs_offset).cast(self._pair_type.pointer()) pairs_start = gdb.Value(start + pairs_offset).cast(self._pair_type.pointer())
src/etc/generate-deriving-span-tests.py
+34 -26
View File
@@ -12,7 +12,8 @@ import os
import stat import stat
TEST_DIR = os.path.abspath( TEST_DIR = os.path.abspath(
os.path.join(os.path.dirname(__file__), '../test/ui/derives/')) os.path.join(os.path.dirname(__file__), "../test/ui/derives/")
)
TEMPLATE = """\ TEMPLATE = """\
// This file was auto-generated using 'src/etc/generate-deriving-span-tests.py' // This file was auto-generated using 'src/etc/generate-deriving-span-tests.py'
@@ -56,28 +57,33 @@ ENUM_TUPLE, ENUM_STRUCT, STRUCT_FIELDS, STRUCT_TUPLE = range(4)
def create_test_case(type, trait, super_traits, error_count): def create_test_case(type, trait, super_traits, error_count):
string = [ENUM_STRING, ENUM_STRUCT_VARIANT_STRING, STRUCT_STRING, STRUCT_TUPLE_STRING][type] string = [
all_traits = ','.join([trait] + super_traits) ENUM_STRING,
super_traits = ','.join(super_traits) ENUM_STRUCT_VARIANT_STRING,
error_deriving = '#[derive(%s)]' % super_traits if super_traits else '' STRUCT_STRING,
STRUCT_TUPLE_STRING,
][type]
all_traits = ",".join([trait] + super_traits)
super_traits = ",".join(super_traits)
error_deriving = "#[derive(%s)]" % super_traits if super_traits else ""
errors = '\n'.join('//~%s ERROR' % ('^' * n) for n in range(error_count)) errors = "\n".join("//~%s ERROR" % ("^" * n) for n in range(error_count))
code = string.format(traits=all_traits, errors=errors) code = string.format(traits=all_traits, errors=errors)
return TEMPLATE.format(error_deriving=error_deriving, code=code) return TEMPLATE.format(error_deriving=error_deriving, code=code)
def write_file(name, string): def write_file(name, string):
test_file = os.path.join(TEST_DIR, 'derives-span-%s.rs' % name) test_file = os.path.join(TEST_DIR, "derives-span-%s.rs" % name)
# set write permission if file exists, so it can be changed # set write permission if file exists, so it can be changed
if os.path.exists(test_file): if os.path.exists(test_file):
os.chmod(test_file, stat.S_IWUSR) os.chmod(test_file, stat.S_IWUSR)
with open(test_file, 'w') as f: with open(test_file, "w") as f:
f.write(string) f.write(string)
# mark file read-only # mark file read-only
os.chmod(test_file, stat.S_IRUSR|stat.S_IRGRP|stat.S_IROTH) os.chmod(test_file, stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
ENUM = 1 ENUM = 1
@@ -85,29 +91,31 @@ STRUCT = 2
ALL = STRUCT | ENUM ALL = STRUCT | ENUM
traits = { traits = {
'Default': (STRUCT, [], 1), "Default": (STRUCT, [], 1),
'FromPrimitive': (0, [], 0), # only works for C-like enums "FromPrimitive": (0, [], 0), # only works for C-like enums
"Decodable": (0, [], 0), # FIXME: quoting gives horrible spans
'Decodable': (0, [], 0), # FIXME: quoting gives horrible spans "Encodable": (0, [], 0), # FIXME: quoting gives horrible spans
'Encodable': (0, [], 0), # FIXME: quoting gives horrible spans
} }
for (trait, supers, errs) in [('Clone', [], 1), for trait, supers, errs in [
('PartialEq', [], 2), ("Clone", [], 1),
('PartialOrd', ['PartialEq'], 1), ("PartialEq", [], 2),
('Eq', ['PartialEq'], 1), ("PartialOrd", ["PartialEq"], 1),
('Ord', ['Eq', 'PartialOrd', 'PartialEq'], 1), ("Eq", ["PartialEq"], 1),
('Debug', [], 1), ("Ord", ["Eq", "PartialOrd", "PartialEq"], 1),
('Hash', [], 1)]: ("Debug", [], 1),
("Hash", [], 1),
]:
traits[trait] = (ALL, supers, errs) traits[trait] = (ALL, supers, errs)
for (trait, (types, super_traits, error_count)) in traits.items(): for trait, (types, super_traits, error_count) in traits.items():
def mk(ty, t=trait, st=super_traits, ec=error_count): def mk(ty, t=trait, st=super_traits, ec=error_count):
return create_test_case(ty, t, st, ec) return create_test_case(ty, t, st, ec)
if types & ENUM: if types & ENUM:
write_file(trait + '-enum', mk(ENUM_TUPLE)) write_file(trait + "-enum", mk(ENUM_TUPLE))
write_file(trait + '-enum-struct-variant', mk(ENUM_STRUCT)) write_file(trait + "-enum-struct-variant", mk(ENUM_STRUCT))
if types & STRUCT: if types & STRUCT:
write_file(trait + '-struct', mk(STRUCT_FIELDS)) write_file(trait + "-struct", mk(STRUCT_FIELDS))
write_file(trait + '-tuple-struct', mk(STRUCT_TUPLE)) write_file(trait + "-tuple-struct", mk(STRUCT_TUPLE))
src/etc/generate-keyword-tests.py
+3 -5
View File
@@ -22,18 +22,16 @@ fn main() {
} }
""" """
test_dir = os.path.abspath( test_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "../test/ui/parser"))
os.path.join(os.path.dirname(__file__), '../test/ui/parser')
)
for kw in sys.argv[1:]: for kw in sys.argv[1:]:
test_file = os.path.join(test_dir, 'keyword-%s-as-identifier.rs' % kw) test_file = os.path.join(test_dir, "keyword-%s-as-identifier.rs" % kw)
# set write permission if file exists, so it can be changed # set write permission if file exists, so it can be changed
if os.path.exists(test_file): if os.path.exists(test_file):
os.chmod(test_file, stat.S_IWUSR) os.chmod(test_file, stat.S_IWUSR)
with open(test_file, 'wt') as f: with open(test_file, "wt") as f:
f.write(template % (kw, kw, kw)) f.write(template % (kw, kw, kw))
# mark file read-only # mark file read-only
src/etc/htmldocck.py
+151 -109
View File
@@ -127,6 +127,7 @@ import os.path
import re import re
import shlex import shlex
from collections import namedtuple from collections import namedtuple
try: try:
from html.parser import HTMLParser from html.parser import HTMLParser
except ImportError: except ImportError:
@@ -142,12 +143,28 @@ except ImportError:
from htmlentitydefs import name2codepoint from htmlentitydefs import name2codepoint
# "void elements" (no closing tag) from the HTML Standard section 12.1.2 # "void elements" (no closing tag) from the HTML Standard section 12.1.2
VOID_ELEMENTS = {'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'keygen', VOID_ELEMENTS = {
'link', 'menuitem', 'meta', 'param', 'source', 'track', 'wbr'} "area",
"base",
"br",
"col",
"embed",
"hr",
"img",
"input",
"keygen",
"link",
"menuitem",
"meta",
"param",
"source",
"track",
"wbr",
}
# Python 2 -> 3 compatibility # Python 2 -> 3 compatibility
try: try:
unichr # noqa: B018 FIXME: py2 unichr # noqa: B018 FIXME: py2
except NameError: except NameError:
unichr = chr unichr = chr
@@ -158,18 +175,20 @@ channel = os.environ["DOC_RUST_LANG_ORG_CHANNEL"]
rust_test_path = None rust_test_path = None
bless = None bless = None
class CustomHTMLParser(HTMLParser): class CustomHTMLParser(HTMLParser):
"""simplified HTML parser. """simplified HTML parser.
this is possible because we are dealing with very regular HTML from this is possible because we are dealing with very regular HTML from
rustdoc; we only have to deal with i) void elements and ii) empty rustdoc; we only have to deal with i) void elements and ii) empty
attributes.""" attributes."""
def __init__(self, target=None): def __init__(self, target=None):
HTMLParser.__init__(self) HTMLParser.__init__(self)
self.__builder = target or ET.TreeBuilder() self.__builder = target or ET.TreeBuilder()
def handle_starttag(self, tag, attrs): def handle_starttag(self, tag, attrs):
attrs = {k: v or '' for k, v in attrs} attrs = {k: v or "" for k, v in attrs}
self.__builder.start(tag, attrs) self.__builder.start(tag, attrs)
if tag in VOID_ELEMENTS: if tag in VOID_ELEMENTS:
self.__builder.end(tag) self.__builder.end(tag)
@@ -178,7 +197,7 @@ class CustomHTMLParser(HTMLParser):
self.__builder.end(tag) self.__builder.end(tag)
def handle_startendtag(self, tag, attrs): def handle_startendtag(self, tag, attrs):
attrs = {k: v or '' for k, v in attrs} attrs = {k: v or "" for k, v in attrs}
self.__builder.start(tag, attrs) self.__builder.start(tag, attrs)
self.__builder.end(tag) self.__builder.end(tag)
@@ -189,7 +208,7 @@ class CustomHTMLParser(HTMLParser):
self.__builder.data(unichr(name2codepoint[name])) self.__builder.data(unichr(name2codepoint[name]))
def handle_charref(self, name): def handle_charref(self, name):
code = int(name[1:], 16) if name.startswith(('x', 'X')) else int(name, 10) code = int(name[1:], 16) if name.startswith(("x", "X")) else int(name, 10)
self.__builder.data(unichr(code)) self.__builder.data(unichr(code))
def close(self): def close(self):
@@ -197,7 +216,7 @@ class CustomHTMLParser(HTMLParser):
return self.__builder.close() return self.__builder.close()
Command = namedtuple('Command', 'negated cmd args lineno context') Command = namedtuple("Command", "negated cmd args lineno context")
class FailedCheck(Exception): class FailedCheck(Exception):
@@ -216,17 +235,17 @@ def concat_multi_lines(f):
concatenated.""" concatenated."""
lastline = None # set to the last line when the last line has a backslash lastline = None # set to the last line when the last line has a backslash
firstlineno = None firstlineno = None
catenated = '' catenated = ""
for lineno, line in enumerate(f): for lineno, line in enumerate(f):
line = line.rstrip('\r\n') line = line.rstrip("\r\n")
# strip the common prefix from the current line if needed # strip the common prefix from the current line if needed
if lastline is not None: if lastline is not None:
common_prefix = os.path.commonprefix([line, lastline]) common_prefix = os.path.commonprefix([line, lastline])
line = line[len(common_prefix):].lstrip() line = line[len(common_prefix) :].lstrip()
firstlineno = firstlineno or lineno firstlineno = firstlineno or lineno
if line.endswith('\\'): if line.endswith("\\"):
if lastline is None: if lastline is None:
lastline = line[:-1] lastline = line[:-1]
catenated += line[:-1] catenated += line[:-1]
@@ -234,10 +253,10 @@ def concat_multi_lines(f):
yield firstlineno, catenated + line yield firstlineno, catenated + line
lastline = None lastline = None
firstlineno = None firstlineno = None
catenated = '' catenated = ""
if lastline is not None: if lastline is not None:
print_err(lineno, line, 'Trailing backslash at the end of the file') print_err(lineno, line, "Trailing backslash at the end of the file")
def get_known_directive_names(): def get_known_directive_names():
@@ -253,12 +272,12 @@ def get_known_directive_names():
"tools/compiletest/src/directive-list.rs", "tools/compiletest/src/directive-list.rs",
), ),
"r", "r",
encoding="utf8" encoding="utf8",
) as fd: ) as fd:
content = fd.read() content = fd.read()
return [ return [
line.strip().replace('",', '').replace('"', '') line.strip().replace('",', "").replace('"', "")
for line in content.split('\n') for line in content.split("\n")
if filter_line(line) if filter_line(line)
] ]
@@ -269,35 +288,42 @@ def get_known_directive_names():
# See <https://github.com/rust-lang/rust/issues/125813#issuecomment-2141953780>. # See <https://github.com/rust-lang/rust/issues/125813#issuecomment-2141953780>.
KNOWN_DIRECTIVE_NAMES = get_known_directive_names() KNOWN_DIRECTIVE_NAMES = get_known_directive_names()
LINE_PATTERN = re.compile(r''' LINE_PATTERN = re.compile(
r"""
//@\s+ //@\s+
(?P<negated>!?)(?P<cmd>[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*) (?P<negated>!?)(?P<cmd>[A-Za-z0-9]+(?:-[A-Za-z0-9]+)*)
(?P<args>.*)$ (?P<args>.*)$
''', re.X | re.UNICODE) """,
re.X | re.UNICODE,
)
def get_commands(template): def get_commands(template):
with io.open(template, encoding='utf-8') as f: with io.open(template, encoding="utf-8") as f:
for lineno, line in concat_multi_lines(f): for lineno, line in concat_multi_lines(f):
m = LINE_PATTERN.search(line) m = LINE_PATTERN.search(line)
if not m: if not m:
continue continue
cmd = m.group('cmd') cmd = m.group("cmd")
negated = (m.group('negated') == '!') negated = m.group("negated") == "!"
if not negated and cmd in KNOWN_DIRECTIVE_NAMES: if not negated and cmd in KNOWN_DIRECTIVE_NAMES:
continue continue
args = m.group('args') args = m.group("args")
if args and not args[:1].isspace(): if args and not args[:1].isspace():
print_err(lineno, line, 'Invalid template syntax') print_err(lineno, line, "Invalid template syntax")
continue continue
try: try:
args = shlex.split(args) args = shlex.split(args)
except UnicodeEncodeError: except UnicodeEncodeError:
args = [arg.decode('utf-8') for arg in shlex.split(args.encode('utf-8'))] args = [
arg.decode("utf-8") for arg in shlex.split(args.encode("utf-8"))
]
except Exception as exc: except Exception as exc:
raise Exception("line {}: {}".format(lineno + 1, exc)) from None raise Exception("line {}: {}".format(lineno + 1, exc)) from None
yield Command(negated=negated, cmd=cmd, args=args, lineno=lineno+1, context=line) yield Command(
negated=negated, cmd=cmd, args=args, lineno=lineno + 1, context=line
)
def _flatten(node, acc): def _flatten(node, acc):
@@ -312,22 +338,24 @@ def _flatten(node, acc):
def flatten(node): def flatten(node):
acc = [] acc = []
_flatten(node, acc) _flatten(node, acc)
return ''.join(acc) return "".join(acc)
def make_xml(text): def make_xml(text):
xml = ET.XML('<xml>%s</xml>' % text) xml = ET.XML("<xml>%s</xml>" % text)
return xml return xml
def normalize_xpath(path): def normalize_xpath(path):
path = path.replace("{{channel}}", channel) path = path.replace("{{channel}}", channel)
if path.startswith('//'): if path.startswith("//"):
return '.' + path # avoid warnings return "." + path # avoid warnings
elif path.startswith('.//'): elif path.startswith(".//"):
return path return path
else: else:
raise InvalidCheck('Non-absolute XPath is not supported due to implementation issues') raise InvalidCheck(
"Non-absolute XPath is not supported due to implementation issues"
)
class CachedFiles(object): class CachedFiles(object):
@@ -338,12 +366,12 @@ class CachedFiles(object):
self.last_path = None self.last_path = None
def resolve_path(self, path): def resolve_path(self, path):
if path != '-': if path != "-":
path = os.path.normpath(path) path = os.path.normpath(path)
self.last_path = path self.last_path = path
return path return path
elif self.last_path is None: elif self.last_path is None:
raise InvalidCheck('Tried to use the previous path in the first command') raise InvalidCheck("Tried to use the previous path in the first command")
else: else:
return self.last_path return self.last_path
@@ -356,10 +384,10 @@ class CachedFiles(object):
return self.files[path] return self.files[path]
abspath = self.get_absolute_path(path) abspath = self.get_absolute_path(path)
if not(os.path.exists(abspath) and os.path.isfile(abspath)): if not (os.path.exists(abspath) and os.path.isfile(abspath)):
raise FailedCheck('File does not exist {!r}'.format(path)) raise FailedCheck("File does not exist {!r}".format(path))
with io.open(abspath, encoding='utf-8') as f: with io.open(abspath, encoding="utf-8") as f:
data = f.read() data = f.read()
self.files[path] = data self.files[path] = data
return data return data
@@ -370,15 +398,15 @@ class CachedFiles(object):
return self.trees[path] return self.trees[path]
abspath = self.get_absolute_path(path) abspath = self.get_absolute_path(path)
if not(os.path.exists(abspath) and os.path.isfile(abspath)): if not (os.path.exists(abspath) and os.path.isfile(abspath)):
raise FailedCheck('File does not exist {!r}'.format(path)) raise FailedCheck("File does not exist {!r}".format(path))
with io.open(abspath, encoding='utf-8') as f: with io.open(abspath, encoding="utf-8") as f:
try: try:
tree = ET.fromstringlist(f.readlines(), CustomHTMLParser()) tree = ET.fromstringlist(f.readlines(), CustomHTMLParser())
except Exception as e: except Exception as e:
raise RuntimeError( # noqa: B904 FIXME: py2 raise RuntimeError( # noqa: B904 FIXME: py2
'Cannot parse an HTML file {!r}: {}'.format(path, e) "Cannot parse an HTML file {!r}: {}".format(path, e)
) )
self.trees[path] = tree self.trees[path] = tree
return self.trees[path] return self.trees[path]
@@ -386,8 +414,8 @@ class CachedFiles(object):
def get_dir(self, path): def get_dir(self, path):
path = self.resolve_path(path) path = self.resolve_path(path)
abspath = self.get_absolute_path(path) abspath = self.get_absolute_path(path)
if not(os.path.exists(abspath) and os.path.isdir(abspath)): if not (os.path.exists(abspath) and os.path.isdir(abspath)):
raise FailedCheck('Directory does not exist {!r}'.format(path)) raise FailedCheck("Directory does not exist {!r}".format(path))
def check_string(data, pat, regexp): def check_string(data, pat, regexp):
@@ -397,8 +425,8 @@ def check_string(data, pat, regexp):
elif regexp: elif regexp:
return re.search(pat, data, flags=re.UNICODE) is not None return re.search(pat, data, flags=re.UNICODE) is not None
else: else:
data = ' '.join(data.split()) data = " ".join(data.split())
pat = ' '.join(pat.split()) pat = " ".join(pat.split())
return pat in data return pat in data
@@ -444,19 +472,19 @@ def get_tree_count(tree, path):
def check_snapshot(snapshot_name, actual_tree, normalize_to_text): def check_snapshot(snapshot_name, actual_tree, normalize_to_text):
assert rust_test_path.endswith('.rs') assert rust_test_path.endswith(".rs")
snapshot_path = '{}.{}.{}'.format(rust_test_path[:-3], snapshot_name, 'html') snapshot_path = "{}.{}.{}".format(rust_test_path[:-3], snapshot_name, "html")
try: try:
with open(snapshot_path, 'r') as snapshot_file: with open(snapshot_path, "r") as snapshot_file:
expected_str = snapshot_file.read().replace("{{channel}}", channel) expected_str = snapshot_file.read().replace("{{channel}}", channel)
except FileNotFoundError: except FileNotFoundError:
if bless: if bless:
expected_str = None expected_str = None
else: else:
raise FailedCheck('No saved snapshot value') # noqa: B904 FIXME: py2 raise FailedCheck("No saved snapshot value") # noqa: B904 FIXME: py2
if not normalize_to_text: if not normalize_to_text:
actual_str = ET.tostring(actual_tree).decode('utf-8') actual_str = ET.tostring(actual_tree).decode("utf-8")
else: else:
actual_str = flatten(actual_tree) actual_str = flatten(actual_tree)
@@ -464,64 +492,66 @@ def check_snapshot(snapshot_name, actual_tree, normalize_to_text):
# 1. Is --bless # 1. Is --bless
# 2. Are actual and expected tree different # 2. Are actual and expected tree different
# 3. Are actual and expected text different # 3. Are actual and expected text different
if not expected_str \ if (
or (not normalize_to_text and \ not expected_str
not compare_tree(make_xml(actual_str), make_xml(expected_str), stderr)) \ or (
or (normalize_to_text and actual_str != expected_str): not normalize_to_text
and not compare_tree(make_xml(actual_str), make_xml(expected_str), stderr)
)
or (normalize_to_text and actual_str != expected_str)
):
if bless: if bless:
with open(snapshot_path, 'w') as snapshot_file: with open(snapshot_path, "w") as snapshot_file:
actual_str = actual_str.replace(channel, "{{channel}}") actual_str = actual_str.replace(channel, "{{channel}}")
snapshot_file.write(actual_str) snapshot_file.write(actual_str)
else: else:
print('--- expected ---\n') print("--- expected ---\n")
print(expected_str) print(expected_str)
print('\n\n--- actual ---\n') print("\n\n--- actual ---\n")
print(actual_str) print(actual_str)
print() print()
raise FailedCheck('Actual snapshot value is different than expected') raise FailedCheck("Actual snapshot value is different than expected")
# Adapted from https://github.com/formencode/formencode/blob/3a1ba9de2fdd494dd945510a4568a3afeddb0b2e/formencode/doctest_xml_compare.py#L72-L120 # Adapted from https://github.com/formencode/formencode/blob/3a1ba9de2fdd494dd945510a4568a3afeddb0b2e/formencode/doctest_xml_compare.py#L72-L120
def compare_tree(x1, x2, reporter=None): def compare_tree(x1, x2, reporter=None):
if x1.tag != x2.tag: if x1.tag != x2.tag:
if reporter: if reporter:
reporter('Tags do not match: %s and %s' % (x1.tag, x2.tag)) reporter("Tags do not match: %s and %s" % (x1.tag, x2.tag))
return False return False
for name, value in x1.attrib.items(): for name, value in x1.attrib.items():
if x2.attrib.get(name) != value: if x2.attrib.get(name) != value:
if reporter: if reporter:
reporter('Attributes do not match: %s=%r, %s=%r' reporter(
% (name, value, name, x2.attrib.get(name))) "Attributes do not match: %s=%r, %s=%r"
% (name, value, name, x2.attrib.get(name))
)
return False return False
for name in x2.attrib: for name in x2.attrib:
if name not in x1.attrib: if name not in x1.attrib:
if reporter: if reporter:
reporter('x2 has an attribute x1 is missing: %s' reporter("x2 has an attribute x1 is missing: %s" % name)
% name)
return False return False
if not text_compare(x1.text, x2.text): if not text_compare(x1.text, x2.text):
if reporter: if reporter:
reporter('text: %r != %r' % (x1.text, x2.text)) reporter("text: %r != %r" % (x1.text, x2.text))
return False return False
if not text_compare(x1.tail, x2.tail): if not text_compare(x1.tail, x2.tail):
if reporter: if reporter:
reporter('tail: %r != %r' % (x1.tail, x2.tail)) reporter("tail: %r != %r" % (x1.tail, x2.tail))
return False return False
cl1 = list(x1) cl1 = list(x1)
cl2 = list(x2) cl2 = list(x2)
if len(cl1) != len(cl2): if len(cl1) != len(cl2):
if reporter: if reporter:
reporter('children length differs, %i != %i' reporter("children length differs, %i != %i" % (len(cl1), len(cl2)))
% (len(cl1), len(cl2)))
return False return False
i = 0 i = 0
for c1, c2 in zip(cl1, cl2): for c1, c2 in zip(cl1, cl2):
i += 1 i += 1
if not compare_tree(c1, c2, reporter=reporter): if not compare_tree(c1, c2, reporter=reporter):
if reporter: if reporter:
reporter('children %i do not match: %s' reporter("children %i do not match: %s" % (i, c1.tag))
% (i, c1.tag))
return False return False
return True return True
@@ -529,14 +559,14 @@ def compare_tree(x1, x2, reporter=None):
def text_compare(t1, t2): def text_compare(t1, t2):
if not t1 and not t2: if not t1 and not t2:
return True return True
if t1 == '*' or t2 == '*': if t1 == "*" or t2 == "*":
return True return True
return (t1 or '').strip() == (t2 or '').strip() return (t1 or "").strip() == (t2 or "").strip()
def stderr(*args): def stderr(*args):
if sys.version_info.major < 3: if sys.version_info.major < 3:
file = codecs.getwriter('utf-8')(sys.stderr) file = codecs.getwriter("utf-8")(sys.stderr)
else: else:
file = sys.stderr file = sys.stderr
@@ -556,21 +586,25 @@ def print_err(lineno, context, err, message=None):
def get_nb_matching_elements(cache, c, regexp, stop_at_first): def get_nb_matching_elements(cache, c, regexp, stop_at_first):
tree = cache.get_tree(c.args[0]) tree = cache.get_tree(c.args[0])
pat, sep, attr = c.args[1].partition('/@') pat, sep, attr = c.args[1].partition("/@")
if sep: # attribute if sep: # attribute
tree = cache.get_tree(c.args[0]) tree = cache.get_tree(c.args[0])
return check_tree_attr(tree, pat, attr, c.args[2], False) return check_tree_attr(tree, pat, attr, c.args[2], False)
else: # normalized text else: # normalized text
pat = c.args[1] pat = c.args[1]
if pat.endswith('/text()'): if pat.endswith("/text()"):
pat = pat[:-7] pat = pat[:-7]
return check_tree_text(cache.get_tree(c.args[0]), pat, c.args[2], regexp, stop_at_first) return check_tree_text(
cache.get_tree(c.args[0]), pat, c.args[2], regexp, stop_at_first
)
def check_files_in_folder(c, cache, folder, files): def check_files_in_folder(c, cache, folder, files):
files = files.strip() files = files.strip()
if not files.startswith('[') or not files.endswith(']'): if not files.startswith("[") or not files.endswith("]"):
raise InvalidCheck("Expected list as second argument of {} (ie '[]')".format(c.cmd)) raise InvalidCheck(
"Expected list as second argument of {} (ie '[]')".format(c.cmd)
)
folder = cache.get_absolute_path(folder) folder = cache.get_absolute_path(folder)
@@ -592,12 +626,18 @@ def check_files_in_folder(c, cache, folder, files):
error = 0 error = 0
if len(files_set) != 0: if len(files_set) != 0:
print_err(c.lineno, c.context, "Entries not found in folder `{}`: `{}`".format( print_err(
folder, files_set)) c.lineno,
c.context,
"Entries not found in folder `{}`: `{}`".format(folder, files_set),
)
error += 1 error += 1
if len(folder_set) != 0: if len(folder_set) != 0:
print_err(c.lineno, c.context, "Extra entries in folder `{}`: `{}`".format( print_err(
folder, folder_set)) c.lineno,
c.context,
"Extra entries in folder `{}`: `{}`".format(folder, folder_set),
)
error += 1 error += 1
return error == 0 return error == 0
@@ -608,11 +648,11 @@ ERR_COUNT = 0
def check_command(c, cache): def check_command(c, cache):
try: try:
cerr = "" cerr = ""
if c.cmd in ['has', 'hasraw', 'matches', 'matchesraw']: # string test if c.cmd in ["has", "hasraw", "matches", "matchesraw"]: # string test
regexp = c.cmd.startswith('matches') regexp = c.cmd.startswith("matches")
# has <path> = file existence # has <path> = file existence
if len(c.args) == 1 and not regexp and 'raw' not in c.cmd: if len(c.args) == 1 and not regexp and "raw" not in c.cmd:
try: try:
cache.get_file(c.args[0]) cache.get_file(c.args[0])
ret = True ret = True
@@ -620,24 +660,24 @@ def check_command(c, cache):
cerr = str(err) cerr = str(err)
ret = False ret = False
# hasraw/matchesraw <path> <pat> = string test # hasraw/matchesraw <path> <pat> = string test
elif len(c.args) == 2 and 'raw' in c.cmd: elif len(c.args) == 2 and "raw" in c.cmd:
cerr = "`PATTERN` did not match" cerr = "`PATTERN` did not match"
ret = check_string(cache.get_file(c.args[0]), c.args[1], regexp) ret = check_string(cache.get_file(c.args[0]), c.args[1], regexp)
# has/matches <path> <pat> <match> = XML tree test # has/matches <path> <pat> <match> = XML tree test
elif len(c.args) == 3 and 'raw' not in c.cmd: elif len(c.args) == 3 and "raw" not in c.cmd:
cerr = "`XPATH PATTERN` did not match" cerr = "`XPATH PATTERN` did not match"
ret = get_nb_matching_elements(cache, c, regexp, True) != 0 ret = get_nb_matching_elements(cache, c, regexp, True) != 0
else: else:
raise InvalidCheck('Invalid number of {} arguments'.format(c.cmd)) raise InvalidCheck("Invalid number of {} arguments".format(c.cmd))
elif c.cmd == 'files': # check files in given folder elif c.cmd == "files": # check files in given folder
if len(c.args) != 2: # files <folder path> <file list> if len(c.args) != 2: # files <folder path> <file list>
raise InvalidCheck("Invalid number of {} arguments".format(c.cmd)) raise InvalidCheck("Invalid number of {} arguments".format(c.cmd))
elif c.negated: elif c.negated:
raise InvalidCheck("{} doesn't support negative check".format(c.cmd)) raise InvalidCheck("{} doesn't support negative check".format(c.cmd))
ret = check_files_in_folder(c, cache, c.args[0], c.args[1]) ret = check_files_in_folder(c, cache, c.args[0], c.args[1])
elif c.cmd == 'count': # count test elif c.cmd == "count": # count test
if len(c.args) == 3: # count <path> <pat> <count> = count test if len(c.args) == 3: # count <path> <pat> <count> = count test
expected = int(c.args[2]) expected = int(c.args[2])
found = get_tree_count(cache.get_tree(c.args[0]), c.args[1]) found = get_tree_count(cache.get_tree(c.args[0]), c.args[1])
@@ -649,15 +689,15 @@ def check_command(c, cache):
cerr = "Expected {} occurrences but found {}".format(expected, found) cerr = "Expected {} occurrences but found {}".format(expected, found)
ret = found == expected ret = found == expected
else: else:
raise InvalidCheck('Invalid number of {} arguments'.format(c.cmd)) raise InvalidCheck("Invalid number of {} arguments".format(c.cmd))
elif c.cmd == 'snapshot': # snapshot test elif c.cmd == "snapshot": # snapshot test
if len(c.args) == 3: # snapshot <snapshot-name> <html-path> <xpath> if len(c.args) == 3: # snapshot <snapshot-name> <html-path> <xpath>
[snapshot_name, html_path, pattern] = c.args [snapshot_name, html_path, pattern] = c.args
tree = cache.get_tree(html_path) tree = cache.get_tree(html_path)
xpath = normalize_xpath(pattern) xpath = normalize_xpath(pattern)
normalize_to_text = False normalize_to_text = False
if xpath.endswith('/text()'): if xpath.endswith("/text()"):
xpath = xpath[:-7] xpath = xpath[:-7]
normalize_to_text = True normalize_to_text = True
@@ -671,13 +711,15 @@ def check_command(c, cache):
cerr = str(err) cerr = str(err)
ret = False ret = False
elif len(subtrees) == 0: elif len(subtrees) == 0:
raise FailedCheck('XPATH did not match') raise FailedCheck("XPATH did not match")
else: else:
raise FailedCheck('Expected 1 match, but found {}'.format(len(subtrees))) raise FailedCheck(
"Expected 1 match, but found {}".format(len(subtrees))
)
else: else:
raise InvalidCheck('Invalid number of {} arguments'.format(c.cmd)) raise InvalidCheck("Invalid number of {} arguments".format(c.cmd))
elif c.cmd == 'has-dir': # has-dir test elif c.cmd == "has-dir": # has-dir test
if len(c.args) == 1: # has-dir <path> = has-dir test if len(c.args) == 1: # has-dir <path> = has-dir test
try: try:
cache.get_dir(c.args[0]) cache.get_dir(c.args[0])
@@ -686,22 +728,22 @@ def check_command(c, cache):
cerr = str(err) cerr = str(err)
ret = False ret = False
else: else:
raise InvalidCheck('Invalid number of {} arguments'.format(c.cmd)) raise InvalidCheck("Invalid number of {} arguments".format(c.cmd))
elif c.cmd == 'valid-html': elif c.cmd == "valid-html":
raise InvalidCheck('Unimplemented valid-html') raise InvalidCheck("Unimplemented valid-html")
elif c.cmd == 'valid-links': elif c.cmd == "valid-links":
raise InvalidCheck('Unimplemented valid-links') raise InvalidCheck("Unimplemented valid-links")
else: else:
raise InvalidCheck('Unrecognized {}'.format(c.cmd)) raise InvalidCheck("Unrecognized {}".format(c.cmd))
if ret == c.negated: if ret == c.negated:
raise FailedCheck(cerr) raise FailedCheck(cerr)
except FailedCheck as err: except FailedCheck as err:
message = '{}{} check failed'.format('!' if c.negated else '', c.cmd) message = "{}{} check failed".format("!" if c.negated else "", c.cmd)
print_err(c.lineno, c.context, str(err), message) print_err(c.lineno, c.context, str(err), message)
except InvalidCheck as err: except InvalidCheck as err:
print_err(c.lineno, c.context, str(err)) print_err(c.lineno, c.context, str(err))
@@ -713,18 +755,18 @@ def check(target, commands):
check_command(c, cache) check_command(c, cache)
if __name__ == '__main__': if __name__ == "__main__":
if len(sys.argv) not in [3, 4]: if len(sys.argv) not in [3, 4]:
stderr('Usage: {} <doc dir> <template> [--bless]'.format(sys.argv[0])) stderr("Usage: {} <doc dir> <template> [--bless]".format(sys.argv[0]))
raise SystemExit(1) raise SystemExit(1)
rust_test_path = sys.argv[2] rust_test_path = sys.argv[2]
if len(sys.argv) > 3 and sys.argv[3] == '--bless': if len(sys.argv) > 3 and sys.argv[3] == "--bless":
bless = True bless = True
else: else:
# We only support `--bless` at the end of the arguments. # We only support `--bless` at the end of the arguments.
# This assert is to prevent silent failures. # This assert is to prevent silent failures.
assert '--bless' not in sys.argv assert "--bless" not in sys.argv
bless = False bless = False
check(sys.argv[1], get_commands(rust_test_path)) check(sys.argv[1], get_commands(rust_test_path))
if ERR_COUNT: if ERR_COUNT:
src/etc/lldb_batchmode.py
+48 -21
View File
@@ -45,7 +45,7 @@ def normalize_whitespace(s):
def breakpoint_callback(frame, bp_loc, dict): def breakpoint_callback(frame, bp_loc, dict):
"""This callback is registered with every breakpoint and makes sure that the """This callback is registered with every breakpoint and makes sure that the
frame containing the breakpoint location is selected """ frame containing the breakpoint location is selected"""
# HACK(eddyb) print a newline to avoid continuing an unfinished line. # HACK(eddyb) print a newline to avoid continuing an unfinished line.
print("") print("")
@@ -79,7 +79,7 @@ def execute_command(command_interpreter, command):
if res.Succeeded(): if res.Succeeded():
if res.HasResult(): if res.HasResult():
print(normalize_whitespace(res.GetOutput() or ''), end='\n') print(normalize_whitespace(res.GetOutput() or ""), end="\n")
# If the command introduced any breakpoints, make sure to register # If the command introduced any breakpoints, make sure to register
# them with the breakpoint # them with the breakpoint
@@ -89,20 +89,32 @@ def execute_command(command_interpreter, command):
breakpoint_id = new_breakpoints.pop() breakpoint_id = new_breakpoints.pop()
if breakpoint_id in registered_breakpoints: if breakpoint_id in registered_breakpoints:
print_debug("breakpoint with id %s is already registered. Ignoring." % print_debug(
str(breakpoint_id)) "breakpoint with id %s is already registered. Ignoring."
% str(breakpoint_id)
)
else: else:
print_debug("registering breakpoint callback, id = " + str(breakpoint_id)) print_debug(
callback_command = ("breakpoint command add -F breakpoint_callback " + "registering breakpoint callback, id = " + str(breakpoint_id)
str(breakpoint_id)) )
callback_command = (
"breakpoint command add -F breakpoint_callback "
+ str(breakpoint_id)
)
command_interpreter.HandleCommand(callback_command, res) command_interpreter.HandleCommand(callback_command, res)
if res.Succeeded(): if res.Succeeded():
print_debug("successfully registered breakpoint callback, id = " + print_debug(
str(breakpoint_id)) "successfully registered breakpoint callback, id = "
+ str(breakpoint_id)
)
registered_breakpoints.add(breakpoint_id) registered_breakpoints.add(breakpoint_id)
else: else:
print("Error while trying to register breakpoint callback, id = " + print(
str(breakpoint_id) + ", message = " + str(res.GetError())) "Error while trying to register breakpoint callback, id = "
+ str(breakpoint_id)
+ ", message = "
+ str(res.GetError())
)
else: else:
print(res.GetError()) print(res.GetError())
@@ -117,14 +129,16 @@ def start_breakpoint_listener(target):
try: try:
while True: while True:
if listener.WaitForEvent(120, event): if listener.WaitForEvent(120, event):
if lldb.SBBreakpoint.EventIsBreakpointEvent(event) and \ if (
lldb.SBBreakpoint.GetBreakpointEventTypeFromEvent(event) == \ lldb.SBBreakpoint.EventIsBreakpointEvent(event)
lldb.eBreakpointEventTypeAdded: and lldb.SBBreakpoint.GetBreakpointEventTypeFromEvent(event)
== lldb.eBreakpointEventTypeAdded
):
global new_breakpoints global new_breakpoints
breakpoint = lldb.SBBreakpoint.GetBreakpointFromEvent(event) breakpoint = lldb.SBBreakpoint.GetBreakpointFromEvent(event)
print_debug("breakpoint added, id = " + str(breakpoint.id)) print_debug("breakpoint added, id = " + str(breakpoint.id))
new_breakpoints.append(breakpoint.id) new_breakpoints.append(breakpoint.id)
except BaseException: # explicitly catch ctrl+c/sysexit except BaseException: # explicitly catch ctrl+c/sysexit
print_debug("breakpoint listener shutting down") print_debug("breakpoint listener shutting down")
# Start the listener and let it run as a daemon # Start the listener and let it run as a daemon
@@ -133,7 +147,9 @@ def start_breakpoint_listener(target):
listener_thread.start() listener_thread.start()
# Register the listener with the target # Register the listener with the target
target.GetBroadcaster().AddListener(listener, lldb.SBTarget.eBroadcastBitBreakpointChanged) target.GetBroadcaster().AddListener(
listener, lldb.SBTarget.eBroadcastBitBreakpointChanged
)
def start_watchdog(): def start_watchdog():
@@ -159,6 +175,7 @@ def start_watchdog():
watchdog_thread.daemon = True watchdog_thread.daemon = True
watchdog_thread.start() watchdog_thread.start()
#################################################################################################### ####################################################################################################
# ~main # ~main
#################################################################################################### ####################################################################################################
@@ -193,8 +210,14 @@ target_error = lldb.SBError()
target = debugger.CreateTarget(target_path, None, None, True, target_error) target = debugger.CreateTarget(target_path, None, None, True, target_error)
if not target: if not target:
print("Could not create debugging target '" + target_path + "': " + print(
str(target_error) + ". Aborting.", file=sys.stderr) "Could not create debugging target '"
+ target_path
+ "': "
+ str(target_error)
+ ". Aborting.",
file=sys.stderr,
)
sys.exit(1) sys.exit(1)
@@ -204,15 +227,19 @@ start_breakpoint_listener(target)
command_interpreter = debugger.GetCommandInterpreter() command_interpreter = debugger.GetCommandInterpreter()
try: try:
script_file = open(script_path, 'r') script_file = open(script_path, "r")
for line in script_file: for line in script_file:
command = line.strip() command = line.strip()
if command == "run" or command == "r" or re.match("^process\s+launch.*", command): if (
command == "run"
or command == "r"
or re.match("^process\s+launch.*", command)
):
# Before starting to run the program, let the thread sleep a bit, so all # Before starting to run the program, let the thread sleep a bit, so all
# breakpoint added events can be processed # breakpoint added events can be processed
time.sleep(0.5) time.sleep(0.5)
if command != '': if command != "":
execute_command(command_interpreter, command) execute_command(command_interpreter, command)
except IOError as e: except IOError as e:
src/etc/lldb_providers.py
+131 -55
View File
@@ -1,7 +1,12 @@
import sys import sys
from lldb import SBData, SBError, eBasicTypeLong, eBasicTypeUnsignedLong, \ from lldb import (
eBasicTypeUnsignedChar SBData,
SBError,
eBasicTypeLong,
eBasicTypeUnsignedLong,
eBasicTypeUnsignedChar,
)
# from lldb.formatters import Logger # from lldb.formatters import Logger
@@ -50,13 +55,17 @@ class ValueBuilder:
def from_int(self, name, value): def from_int(self, name, value):
# type: (str, int) -> SBValue # type: (str, int) -> SBValue
type = self.valobj.GetType().GetBasicType(eBasicTypeLong) type = self.valobj.GetType().GetBasicType(eBasicTypeLong)
data = SBData.CreateDataFromSInt64Array(self.endianness, self.pointer_size, [value]) data = SBData.CreateDataFromSInt64Array(
self.endianness, self.pointer_size, [value]
)
return self.valobj.CreateValueFromData(name, data, type) return self.valobj.CreateValueFromData(name, data, type)
def from_uint(self, name, value): def from_uint(self, name, value):
# type: (str, int) -> SBValue # type: (str, int) -> SBValue
type = self.valobj.GetType().GetBasicType(eBasicTypeUnsignedLong) type = self.valobj.GetType().GetBasicType(eBasicTypeUnsignedLong)
data = SBData.CreateDataFromUInt64Array(self.endianness, self.pointer_size, [value]) data = SBData.CreateDataFromUInt64Array(
self.endianness, self.pointer_size, [value]
)
return self.valobj.CreateValueFromData(name, data, type) return self.valobj.CreateValueFromData(name, data, type)
@@ -127,13 +136,17 @@ class EmptySyntheticProvider:
def SizeSummaryProvider(valobj, dict): def SizeSummaryProvider(valobj, dict):
# type: (SBValue, dict) -> str # type: (SBValue, dict) -> str
return 'size=' + str(valobj.GetNumChildren()) return "size=" + str(valobj.GetNumChildren())
def vec_to_string(vec): def vec_to_string(vec):
length = vec.GetNumChildren() length = vec.GetNumChildren()
chars = [vec.GetChildAtIndex(i).GetValueAsUnsigned() for i in range(length)] chars = [vec.GetChildAtIndex(i).GetValueAsUnsigned() for i in range(length)]
return bytes(chars).decode(errors='replace') if PY3 else "".join(chr(char) for char in chars) return (
bytes(chars).decode(errors="replace")
if PY3
else "".join(chr(char) for char in chars)
)
def StdStringSummaryProvider(valobj, dict): def StdStringSummaryProvider(valobj, dict):
@@ -172,7 +185,7 @@ def StdStrSummaryProvider(valobj, dict):
error = SBError() error = SBError()
process = data_ptr.GetProcess() process = data_ptr.GetProcess()
data = process.ReadMemory(start, length, error) data = process.ReadMemory(start, length, error)
data = data.decode(encoding='UTF-8') if PY3 else data data = data.decode(encoding="UTF-8") if PY3 else data
return '"%s"' % data return '"%s"' % data
@@ -199,9 +212,9 @@ def StdPathSummaryProvider(valobj, dict):
data = process.ReadMemory(start, length, error) data = process.ReadMemory(start, length, error)
if PY3: if PY3:
try: try:
data = data.decode(encoding='UTF-8') data = data.decode(encoding="UTF-8")
except UnicodeDecodeError: except UnicodeDecodeError:
return '%r' % data return "%r" % data
return '"%s"' % data return '"%s"' % data
@@ -250,8 +263,10 @@ class StructSyntheticProvider:
# type: () -> bool # type: () -> bool
return True return True
class ClangEncodedEnumProvider: class ClangEncodedEnumProvider:
"""Pretty-printer for 'clang-encoded' enums support implemented in LLDB""" """Pretty-printer for 'clang-encoded' enums support implemented in LLDB"""
DISCRIMINANT_MEMBER_NAME = "$discr$" DISCRIMINANT_MEMBER_NAME = "$discr$"
VALUE_MEMBER_NAME = "value" VALUE_MEMBER_NAME = "value"
@@ -260,7 +275,7 @@ class ClangEncodedEnumProvider:
self.update() self.update()
def has_children(self): def has_children(self):
return True return True
def num_children(self): def num_children(self):
if self.is_default: if self.is_default:
@@ -276,25 +291,32 @@ class ClangEncodedEnumProvider:
def get_child_at_index(self, index): def get_child_at_index(self, index):
if index == 0: if index == 0:
return self.variant.GetChildMemberWithName(ClangEncodedEnumProvider.VALUE_MEMBER_NAME) return self.variant.GetChildMemberWithName(
ClangEncodedEnumProvider.VALUE_MEMBER_NAME
)
if index == 1: if index == 1:
return self.variant.GetChildMemberWithName( return self.variant.GetChildMemberWithName(
ClangEncodedEnumProvider.DISCRIMINANT_MEMBER_NAME) ClangEncodedEnumProvider.DISCRIMINANT_MEMBER_NAME
)
def update(self): def update(self):
all_variants = self.valobj.GetChildAtIndex(0) all_variants = self.valobj.GetChildAtIndex(0)
index = self._getCurrentVariantIndex(all_variants) index = self._getCurrentVariantIndex(all_variants)
self.variant = all_variants.GetChildAtIndex(index) self.variant = all_variants.GetChildAtIndex(index)
self.is_default = self.variant.GetIndexOfChildWithName( self.is_default = (
ClangEncodedEnumProvider.DISCRIMINANT_MEMBER_NAME) == -1 self.variant.GetIndexOfChildWithName(
ClangEncodedEnumProvider.DISCRIMINANT_MEMBER_NAME
)
== -1
)
def _getCurrentVariantIndex(self, all_variants): def _getCurrentVariantIndex(self, all_variants):
default_index = 0 default_index = 0
for i in range(all_variants.GetNumChildren()): for i in range(all_variants.GetNumChildren()):
variant = all_variants.GetChildAtIndex(i) variant = all_variants.GetChildAtIndex(i)
discr = variant.GetChildMemberWithName( discr = variant.GetChildMemberWithName(
ClangEncodedEnumProvider.DISCRIMINANT_MEMBER_NAME) ClangEncodedEnumProvider.DISCRIMINANT_MEMBER_NAME
)
if discr.IsValid(): if discr.IsValid():
discr_unsigned_value = discr.GetValueAsUnsigned() discr_unsigned_value = discr.GetValueAsUnsigned()
if variant.GetName() == f"$variant${discr_unsigned_value}": if variant.GetName() == f"$variant${discr_unsigned_value}":
@@ -303,6 +325,7 @@ class ClangEncodedEnumProvider:
default_index = i default_index = i
return default_index return default_index
class TupleSyntheticProvider: class TupleSyntheticProvider:
"""Pretty-printer for tuples and tuple enum variants""" """Pretty-printer for tuples and tuple enum variants"""
@@ -336,7 +359,9 @@ class TupleSyntheticProvider:
else: else:
field = self.type.GetFieldAtIndex(index) field = self.type.GetFieldAtIndex(index)
element = self.valobj.GetChildMemberWithName(field.name) element = self.valobj.GetChildMemberWithName(field.name)
return self.valobj.CreateValueFromData(str(index), element.GetData(), element.GetType()) return self.valobj.CreateValueFromData(
str(index), element.GetData(), element.GetType()
)
def update(self): def update(self):
# type: () -> None # type: () -> None
@@ -373,7 +398,7 @@ class StdVecSyntheticProvider:
def get_child_index(self, name): def get_child_index(self, name):
# type: (str) -> int # type: (str) -> int
index = name.lstrip('[').rstrip(']') index = name.lstrip("[").rstrip("]")
if index.isdigit(): if index.isdigit():
return int(index) return int(index)
else: else:
@@ -383,15 +408,21 @@ class StdVecSyntheticProvider:
# type: (int) -> SBValue # type: (int) -> SBValue
start = self.data_ptr.GetValueAsUnsigned() start = self.data_ptr.GetValueAsUnsigned()
address = start + index * self.element_type_size address = start + index * self.element_type_size
element = self.data_ptr.CreateValueFromAddress("[%s]" % index, address, self.element_type) element = self.data_ptr.CreateValueFromAddress(
"[%s]" % index, address, self.element_type
)
return element return element
def update(self): def update(self):
# type: () -> None # type: () -> None
self.length = self.valobj.GetChildMemberWithName("len").GetValueAsUnsigned() self.length = self.valobj.GetChildMemberWithName("len").GetValueAsUnsigned()
self.buf = self.valobj.GetChildMemberWithName("buf").GetChildMemberWithName("inner") self.buf = self.valobj.GetChildMemberWithName("buf").GetChildMemberWithName(
"inner"
)
self.data_ptr = unwrap_unique_or_non_null(self.buf.GetChildMemberWithName("ptr")) self.data_ptr = unwrap_unique_or_non_null(
self.buf.GetChildMemberWithName("ptr")
)
self.element_type = self.valobj.GetType().GetTemplateArgumentType(0) self.element_type = self.valobj.GetType().GetTemplateArgumentType(0)
self.element_type_size = self.element_type.GetByteSize() self.element_type_size = self.element_type.GetByteSize()
@@ -412,7 +443,7 @@ class StdSliceSyntheticProvider:
def get_child_index(self, name): def get_child_index(self, name):
# type: (str) -> int # type: (str) -> int
index = name.lstrip('[').rstrip(']') index = name.lstrip("[").rstrip("]")
if index.isdigit(): if index.isdigit():
return int(index) return int(index)
else: else:
@@ -422,7 +453,9 @@ class StdSliceSyntheticProvider:
# type: (int) -> SBValue # type: (int) -> SBValue
start = self.data_ptr.GetValueAsUnsigned() start = self.data_ptr.GetValueAsUnsigned()
address = start + index * self.element_type_size address = start + index * self.element_type_size
element = self.data_ptr.CreateValueFromAddress("[%s]" % index, address, self.element_type) element = self.data_ptr.CreateValueFromAddress(
"[%s]" % index, address, self.element_type
)
return element return element
def update(self): def update(self):
@@ -457,7 +490,7 @@ class StdVecDequeSyntheticProvider:
def get_child_index(self, name): def get_child_index(self, name):
# type: (str) -> int # type: (str) -> int
index = name.lstrip('[').rstrip(']') index = name.lstrip("[").rstrip("]")
if index.isdigit() and int(index) < self.size: if index.isdigit() and int(index) < self.size:
return int(index) return int(index)
else: else:
@@ -467,20 +500,26 @@ class StdVecDequeSyntheticProvider:
# type: (int) -> SBValue # type: (int) -> SBValue
start = self.data_ptr.GetValueAsUnsigned() start = self.data_ptr.GetValueAsUnsigned()
address = start + ((index + self.head) % self.cap) * self.element_type_size address = start + ((index + self.head) % self.cap) * self.element_type_size
element = self.data_ptr.CreateValueFromAddress("[%s]" % index, address, self.element_type) element = self.data_ptr.CreateValueFromAddress(
"[%s]" % index, address, self.element_type
)
return element return element
def update(self): def update(self):
# type: () -> None # type: () -> None
self.head = self.valobj.GetChildMemberWithName("head").GetValueAsUnsigned() self.head = self.valobj.GetChildMemberWithName("head").GetValueAsUnsigned()
self.size = self.valobj.GetChildMemberWithName("len").GetValueAsUnsigned() self.size = self.valobj.GetChildMemberWithName("len").GetValueAsUnsigned()
self.buf = self.valobj.GetChildMemberWithName("buf").GetChildMemberWithName("inner") self.buf = self.valobj.GetChildMemberWithName("buf").GetChildMemberWithName(
"inner"
)
cap = self.buf.GetChildMemberWithName("cap") cap = self.buf.GetChildMemberWithName("cap")
if cap.GetType().num_fields == 1: if cap.GetType().num_fields == 1:
cap = cap.GetChildAtIndex(0) cap = cap.GetChildAtIndex(0)
self.cap = cap.GetValueAsUnsigned() self.cap = cap.GetValueAsUnsigned()
self.data_ptr = unwrap_unique_or_non_null(self.buf.GetChildMemberWithName("ptr")) self.data_ptr = unwrap_unique_or_non_null(
self.buf.GetChildMemberWithName("ptr")
)
self.element_type = self.valobj.GetType().GetTemplateArgumentType(0) self.element_type = self.valobj.GetType().GetTemplateArgumentType(0)
self.element_type_size = self.element_type.GetByteSize() self.element_type_size = self.element_type.GetByteSize()
@@ -510,7 +549,7 @@ class StdOldHashMapSyntheticProvider:
def get_child_index(self, name): def get_child_index(self, name):
# type: (str) -> int # type: (str) -> int
index = name.lstrip('[').rstrip(']') index = name.lstrip("[").rstrip("]")
if index.isdigit(): if index.isdigit():
return int(index) return int(index)
else: else:
@@ -525,8 +564,14 @@ class StdOldHashMapSyntheticProvider:
hashes = self.hash_uint_size * self.capacity hashes = self.hash_uint_size * self.capacity
align = self.pair_type_size align = self.pair_type_size
# See `libcore/alloc.rs:padding_needed_for` # See `libcore/alloc.rs:padding_needed_for`
len_rounded_up = (((((hashes + align) % self.modulo - 1) % self.modulo) & ~( len_rounded_up = (
(align - 1) % self.modulo)) % self.modulo - hashes) % self.modulo (
(((hashes + align) % self.modulo - 1) % self.modulo)
& ~((align - 1) % self.modulo)
)
% self.modulo
- hashes
) % self.modulo
# len_rounded_up = ((hashes + align - 1) & ~(align - 1)) - hashes # len_rounded_up = ((hashes + align - 1) & ~(align - 1)) - hashes
pairs_offset = hashes + len_rounded_up pairs_offset = hashes + len_rounded_up
@@ -535,12 +580,16 @@ class StdOldHashMapSyntheticProvider:
table_index = self.valid_indices[index] table_index = self.valid_indices[index]
idx = table_index & self.capacity_mask idx = table_index & self.capacity_mask
address = pairs_start + idx * self.pair_type_size address = pairs_start + idx * self.pair_type_size
element = self.data_ptr.CreateValueFromAddress("[%s]" % index, address, self.pair_type) element = self.data_ptr.CreateValueFromAddress(
"[%s]" % index, address, self.pair_type
)
if self.show_values: if self.show_values:
return element return element
else: else:
key = element.GetChildAtIndex(0) key = element.GetChildAtIndex(0)
return self.valobj.CreateValueFromData("[%s]" % index, key.GetData(), key.GetType()) return self.valobj.CreateValueFromData(
"[%s]" % index, key.GetData(), key.GetType()
)
def update(self): def update(self):
# type: () -> None # type: () -> None
@@ -551,10 +600,12 @@ class StdOldHashMapSyntheticProvider:
self.hashes = self.table.GetChildMemberWithName("hashes") self.hashes = self.table.GetChildMemberWithName("hashes")
self.hash_uint_type = self.hashes.GetType() self.hash_uint_type = self.hashes.GetType()
self.hash_uint_size = self.hashes.GetType().GetByteSize() self.hash_uint_size = self.hashes.GetType().GetByteSize()
self.modulo = 2 ** self.hash_uint_size self.modulo = 2**self.hash_uint_size
self.data_ptr = self.hashes.GetChildAtIndex(0).GetChildAtIndex(0) self.data_ptr = self.hashes.GetChildAtIndex(0).GetChildAtIndex(0)
self.capacity_mask = self.table.GetChildMemberWithName("capacity_mask").GetValueAsUnsigned() self.capacity_mask = self.table.GetChildMemberWithName(
"capacity_mask"
).GetValueAsUnsigned()
self.capacity = (self.capacity_mask + 1) % self.modulo self.capacity = (self.capacity_mask + 1) % self.modulo
marker = self.table.GetChildMemberWithName("marker").GetType() # type: SBType marker = self.table.GetChildMemberWithName("marker").GetType() # type: SBType
@@ -564,8 +615,9 @@ class StdOldHashMapSyntheticProvider:
self.valid_indices = [] self.valid_indices = []
for idx in range(self.capacity): for idx in range(self.capacity):
address = self.data_ptr.GetValueAsUnsigned() + idx * self.hash_uint_size address = self.data_ptr.GetValueAsUnsigned() + idx * self.hash_uint_size
hash_uint = self.data_ptr.CreateValueFromAddress("[%s]" % idx, address, hash_uint = self.data_ptr.CreateValueFromAddress(
self.hash_uint_type) "[%s]" % idx, address, self.hash_uint_type
)
hash_ptr = hash_uint.GetChildAtIndex(0).GetChildAtIndex(0) hash_ptr = hash_uint.GetChildAtIndex(0).GetChildAtIndex(0)
if hash_ptr.GetValueAsUnsigned() != 0: if hash_ptr.GetValueAsUnsigned() != 0:
self.valid_indices.append(idx) self.valid_indices.append(idx)
@@ -592,7 +644,7 @@ class StdHashMapSyntheticProvider:
def get_child_index(self, name): def get_child_index(self, name):
# type: (str) -> int # type: (str) -> int
index = name.lstrip('[').rstrip(']') index = name.lstrip("[").rstrip("]")
if index.isdigit(): if index.isdigit():
return int(index) return int(index)
else: else:
@@ -605,19 +657,25 @@ class StdHashMapSyntheticProvider:
if self.new_layout: if self.new_layout:
idx = -(idx + 1) idx = -(idx + 1)
address = pairs_start + idx * self.pair_type_size address = pairs_start + idx * self.pair_type_size
element = self.data_ptr.CreateValueFromAddress("[%s]" % index, address, self.pair_type) element = self.data_ptr.CreateValueFromAddress(
"[%s]" % index, address, self.pair_type
)
if self.show_values: if self.show_values:
return element return element
else: else:
key = element.GetChildAtIndex(0) key = element.GetChildAtIndex(0)
return self.valobj.CreateValueFromData("[%s]" % index, key.GetData(), key.GetType()) return self.valobj.CreateValueFromData(
"[%s]" % index, key.GetData(), key.GetType()
)
def update(self): def update(self):
# type: () -> None # type: () -> None
table = self.table() table = self.table()
inner_table = table.GetChildMemberWithName("table") inner_table = table.GetChildMemberWithName("table")
capacity = inner_table.GetChildMemberWithName("bucket_mask").GetValueAsUnsigned() + 1 capacity = (
inner_table.GetChildMemberWithName("bucket_mask").GetValueAsUnsigned() + 1
)
ctrl = inner_table.GetChildMemberWithName("ctrl").GetChildAtIndex(0) ctrl = inner_table.GetChildMemberWithName("ctrl").GetChildAtIndex(0)
self.size = inner_table.GetChildMemberWithName("items").GetValueAsUnsigned() self.size = inner_table.GetChildMemberWithName("items").GetValueAsUnsigned()
@@ -630,16 +688,21 @@ class StdHashMapSyntheticProvider:
if self.new_layout: if self.new_layout:
self.data_ptr = ctrl.Cast(self.pair_type.GetPointerType()) self.data_ptr = ctrl.Cast(self.pair_type.GetPointerType())
else: else:
self.data_ptr = inner_table.GetChildMemberWithName("data").GetChildAtIndex(0) self.data_ptr = inner_table.GetChildMemberWithName("data").GetChildAtIndex(
0
)
u8_type = self.valobj.GetTarget().GetBasicType(eBasicTypeUnsignedChar) u8_type = self.valobj.GetTarget().GetBasicType(eBasicTypeUnsignedChar)
u8_type_size = self.valobj.GetTarget().GetBasicType(eBasicTypeUnsignedChar).GetByteSize() u8_type_size = (
self.valobj.GetTarget().GetBasicType(eBasicTypeUnsignedChar).GetByteSize()
)
self.valid_indices = [] self.valid_indices = []
for idx in range(capacity): for idx in range(capacity):
address = ctrl.GetValueAsUnsigned() + idx * u8_type_size address = ctrl.GetValueAsUnsigned() + idx * u8_type_size
value = ctrl.CreateValueFromAddress("ctrl[%s]" % idx, address, value = ctrl.CreateValueFromAddress(
u8_type).GetValueAsUnsigned() "ctrl[%s]" % idx, address, u8_type
).GetValueAsUnsigned()
is_present = value & 128 == 0 is_present = value & 128 == 0
if is_present: if is_present:
self.valid_indices.append(idx) self.valid_indices.append(idx)
@@ -691,10 +754,16 @@ class StdRcSyntheticProvider:
self.value = self.ptr.GetChildMemberWithName("data" if is_atomic else "value") self.value = self.ptr.GetChildMemberWithName("data" if is_atomic else "value")
self.strong = self.ptr.GetChildMemberWithName("strong").GetChildAtIndex( self.strong = (
0).GetChildMemberWithName("value") self.ptr.GetChildMemberWithName("strong")
self.weak = self.ptr.GetChildMemberWithName("weak").GetChildAtIndex( .GetChildAtIndex(0)
0).GetChildMemberWithName("value") .GetChildMemberWithName("value")
)
self.weak = (
self.ptr.GetChildMemberWithName("weak")
.GetChildAtIndex(0)
.GetChildMemberWithName("value")
)
self.value_builder = ValueBuilder(valobj) self.value_builder = ValueBuilder(valobj)
@@ -772,7 +841,9 @@ class StdCellSyntheticProvider:
def StdRefSummaryProvider(valobj, dict): def StdRefSummaryProvider(valobj, dict):
# type: (SBValue, dict) -> str # type: (SBValue, dict) -> str
borrow = valobj.GetChildMemberWithName("borrow").GetValueAsSigned() borrow = valobj.GetChildMemberWithName("borrow").GetValueAsSigned()
return "borrow={}".format(borrow) if borrow >= 0 else "borrow_mut={}".format(-borrow) return (
"borrow={}".format(borrow) if borrow >= 0 else "borrow_mut={}".format(-borrow)
)
class StdRefSyntheticProvider: class StdRefSyntheticProvider:
@@ -785,11 +856,16 @@ class StdRefSyntheticProvider:
borrow = valobj.GetChildMemberWithName("borrow") borrow = valobj.GetChildMemberWithName("borrow")
value = valobj.GetChildMemberWithName("value") value = valobj.GetChildMemberWithName("value")
if is_cell: if is_cell:
self.borrow = borrow.GetChildMemberWithName("value").GetChildMemberWithName("value") self.borrow = borrow.GetChildMemberWithName("value").GetChildMemberWithName(
"value"
)
self.value = value.GetChildMemberWithName("value") self.value = value.GetChildMemberWithName("value")
else: else:
self.borrow = borrow.GetChildMemberWithName("borrow").GetChildMemberWithName( self.borrow = (
"value").GetChildMemberWithName("value") borrow.GetChildMemberWithName("borrow")
.GetChildMemberWithName("value")
.GetChildMemberWithName("value")
)
self.value = value.Dereference() self.value = value.Dereference()
self.value_builder = ValueBuilder(valobj) self.value_builder = ValueBuilder(valobj)
@@ -832,7 +908,7 @@ def StdNonZeroNumberSummaryProvider(valobj, _dict):
# FIXME: Avoid printing as character literal, # FIXME: Avoid printing as character literal,
# see https://github.com/llvm/llvm-project/issues/65076. # see https://github.com/llvm/llvm-project/issues/65076.
if inner_inner.GetTypeName() in ['char', 'unsigned char']: if inner_inner.GetTypeName() in ["char", "unsigned char"]:
return str(inner_inner.GetValueAsSigned()) return str(inner_inner.GetValueAsSigned())
else: else:
return inner_inner.GetValue() return inner_inner.GetValue()
src/etc/rust_types.py
+2 -1
View File
@@ -54,7 +54,7 @@ STD_REF_MUT_REGEX = re.compile(r"^(core::([a-z_]+::)+)RefMut<.+>$")
STD_REF_CELL_REGEX = re.compile(r"^(core::([a-z_]+::)+)RefCell<.+>$") STD_REF_CELL_REGEX = re.compile(r"^(core::([a-z_]+::)+)RefCell<.+>$")
STD_NONZERO_NUMBER_REGEX = re.compile(r"^(core::([a-z_]+::)+)NonZero<.+>$") STD_NONZERO_NUMBER_REGEX = re.compile(r"^(core::([a-z_]+::)+)NonZero<.+>$")
STD_PATHBUF_REGEX = re.compile(r"^(std::([a-z_]+::)+)PathBuf$") STD_PATHBUF_REGEX = re.compile(r"^(std::([a-z_]+::)+)PathBuf$")
STD_PATH_REGEX = re.compile(r"^&(mut )?(std::([a-z_]+::)+)Path$") STD_PATH_REGEX = re.compile(r"^&(mut )?(std::([a-z_]+::)+)Path$")
TUPLE_ITEM_REGEX = re.compile(r"__\d+$") TUPLE_ITEM_REGEX = re.compile(r"__\d+$")
@@ -84,6 +84,7 @@ STD_TYPE_TO_REGEX = {
RustType.STD_PATH: STD_PATH_REGEX, RustType.STD_PATH: STD_PATH_REGEX,
} }
def is_tuple_fields(fields): def is_tuple_fields(fields):
# type: (list) -> bool # type: (list) -> bool
return all(TUPLE_ITEM_REGEX.match(str(field.name)) for field in fields) return all(TUPLE_ITEM_REGEX.match(str(field.name)) for field in fields)
src/tools/publish_toolstate.py
+125 -101
View File
@@ -14,6 +14,7 @@ import json
import datetime import datetime
import collections import collections
import textwrap import textwrap
try: try:
import urllib2 import urllib2
from urllib2 import HTTPError from urllib2 import HTTPError
@@ -21,7 +22,7 @@ except ImportError:
import urllib.request as urllib2 import urllib.request as urllib2
from urllib.error import HTTPError from urllib.error import HTTPError
try: try:
import typing # noqa: F401 FIXME: py2 import typing # noqa: F401 FIXME: py2
except ImportError: except ImportError:
pass pass
@@ -29,40 +30,41 @@ except ImportError:
# These should be collaborators of the rust-lang/rust repository (with at least # These should be collaborators of the rust-lang/rust repository (with at least
# read privileges on it). CI will fail otherwise. # read privileges on it). CI will fail otherwise.
MAINTAINERS = { MAINTAINERS = {
'book': {'carols10cents'}, "book": {"carols10cents"},
'nomicon': {'frewsxcv', 'Gankra', 'JohnTitor'}, "nomicon": {"frewsxcv", "Gankra", "JohnTitor"},
'reference': {'Havvy', 'matthewjasper', 'ehuss'}, "reference": {"Havvy", "matthewjasper", "ehuss"},
'rust-by-example': {'marioidival'}, "rust-by-example": {"marioidival"},
'embedded-book': {'adamgreig', 'andre-richter', 'jamesmunns', 'therealprof'}, "embedded-book": {"adamgreig", "andre-richter", "jamesmunns", "therealprof"},
'edition-guide': {'ehuss'}, "edition-guide": {"ehuss"},
'rustc-dev-guide': {'spastorino', 'amanjeev', 'JohnTitor'}, "rustc-dev-guide": {"spastorino", "amanjeev", "JohnTitor"},
} }
LABELS = { LABELS = {
'book': ['C-bug'], "book": ["C-bug"],
'nomicon': ['C-bug'], "nomicon": ["C-bug"],
'reference': ['C-bug'], "reference": ["C-bug"],
'rust-by-example': ['C-bug'], "rust-by-example": ["C-bug"],
'embedded-book': ['C-bug'], "embedded-book": ["C-bug"],
'edition-guide': ['C-bug'], "edition-guide": ["C-bug"],
'rustc-dev-guide': ['C-bug'], "rustc-dev-guide": ["C-bug"],
} }
REPOS = { REPOS = {
'book': 'https://github.com/rust-lang/book', "book": "https://github.com/rust-lang/book",
'nomicon': 'https://github.com/rust-lang/nomicon', "nomicon": "https://github.com/rust-lang/nomicon",
'reference': 'https://github.com/rust-lang/reference', "reference": "https://github.com/rust-lang/reference",
'rust-by-example': 'https://github.com/rust-lang/rust-by-example', "rust-by-example": "https://github.com/rust-lang/rust-by-example",
'embedded-book': 'https://github.com/rust-embedded/book', "embedded-book": "https://github.com/rust-embedded/book",
'edition-guide': 'https://github.com/rust-lang/edition-guide', "edition-guide": "https://github.com/rust-lang/edition-guide",
'rustc-dev-guide': 'https://github.com/rust-lang/rustc-dev-guide', "rustc-dev-guide": "https://github.com/rust-lang/rustc-dev-guide",
} }
def load_json_from_response(resp): def load_json_from_response(resp):
# type: (typing.Any) -> typing.Any # type: (typing.Any) -> typing.Any
content = resp.read() content = resp.read()
if isinstance(content, bytes): if isinstance(content, bytes):
content_str = content.decode('utf-8') content_str = content.decode("utf-8")
else: else:
print("Refusing to decode " + str(type(content)) + " to str") print("Refusing to decode " + str(type(content)) + " to str")
return json.loads(content_str) return json.loads(content_str)
@@ -70,11 +72,10 @@ def load_json_from_response(resp):
def read_current_status(current_commit, path): def read_current_status(current_commit, path):
# type: (str, str) -> typing.Mapping[str, typing.Any] # type: (str, str) -> typing.Mapping[str, typing.Any]
'''Reads build status of `current_commit` from content of `history/*.tsv` """Reads build status of `current_commit` from content of `history/*.tsv`"""
''' with open(path, "r") as f:
with open(path, 'r') as f:
for line in f: for line in f:
(commit, status) = line.split('\t', 1) (commit, status) = line.split("\t", 1)
if commit == current_commit: if commit == current_commit:
return json.loads(status) return json.loads(status)
return {} return {}
@@ -82,12 +83,12 @@ def read_current_status(current_commit, path):
def gh_url(): def gh_url():
# type: () -> str # type: () -> str
return os.environ['TOOLSTATE_ISSUES_API_URL'] return os.environ["TOOLSTATE_ISSUES_API_URL"]
def maybe_remove_mention(message): def maybe_remove_mention(message):
# type: (str) -> str # type: (str) -> str
if os.environ.get('TOOLSTATE_SKIP_MENTIONS') is not None: if os.environ.get("TOOLSTATE_SKIP_MENTIONS") is not None:
return message.replace("@", "") return message.replace("@", "")
return message return message
@@ -102,36 +103,45 @@ def issue(
github_token, github_token,
): ):
# type: (str, str, typing.Iterable[str], str, str, typing.List[str], str) -> None # type: (str, str, typing.Iterable[str], str, str, typing.List[str], str) -> None
'''Open an issue about the toolstate failure.''' """Open an issue about the toolstate failure."""
if status == 'test-fail': if status == "test-fail":
status_description = 'has failing tests' status_description = "has failing tests"
else: else:
status_description = 'no longer builds' status_description = "no longer builds"
request = json.dumps({ request = json.dumps(
'body': maybe_remove_mention(textwrap.dedent('''\ {
"body": maybe_remove_mention(
textwrap.dedent("""\
Hello, this is your friendly neighborhood mergebot. Hello, this is your friendly neighborhood mergebot.
After merging PR {}, I observed that the tool {} {}. After merging PR {}, I observed that the tool {} {}.
A follow-up PR to the repository {} is needed to fix the fallout. A follow-up PR to the repository {} is needed to fix the fallout.
cc @{}, do you think you would have time to do the follow-up work? cc @{}, do you think you would have time to do the follow-up work?
If so, that would be great! If so, that would be great!
''').format( """).format(
relevant_pr_number, tool, status_description, relevant_pr_number,
REPOS.get(tool), relevant_pr_user tool,
)), status_description,
'title': '`{}` no longer builds after {}'.format(tool, relevant_pr_number), REPOS.get(tool),
'assignees': list(assignees), relevant_pr_user,
'labels': labels, )
}) ),
print("Creating issue:\n{}".format(request)) "title": "`{}` no longer builds after {}".format(tool, relevant_pr_number),
response = urllib2.urlopen(urllib2.Request( "assignees": list(assignees),
gh_url(), "labels": labels,
request.encode(),
{
'Authorization': 'token ' + github_token,
'Content-Type': 'application/json',
} }
)) )
print("Creating issue:\n{}".format(request))
response = urllib2.urlopen(
urllib2.Request(
gh_url(),
request.encode(),
{
"Authorization": "token " + github_token,
"Content-Type": "application/json",
},
)
)
response.read() response.read()
@@ -145,27 +155,26 @@ def update_latest(
github_token, github_token,
): ):
# type: (str, str, str, str, str, str, str) -> str # type: (str, str, str, str, str, str, str) -> str
'''Updates `_data/latest.json` to match build result of the given commit. """Updates `_data/latest.json` to match build result of the given commit."""
''' with open("_data/latest.json", "r+") as f:
with open('_data/latest.json', 'r+') as f:
latest = json.load(f, object_pairs_hook=collections.OrderedDict) latest = json.load(f, object_pairs_hook=collections.OrderedDict)
current_status = { current_status = {
os_: read_current_status(current_commit, 'history/' + os_ + '.tsv') os_: read_current_status(current_commit, "history/" + os_ + ".tsv")
for os_ in ['windows', 'linux'] for os_ in ["windows", "linux"]
} }
slug = 'rust-lang/rust' slug = "rust-lang/rust"
message = textwrap.dedent('''\ message = textwrap.dedent("""\
📣 Toolstate changed by {}! 📣 Toolstate changed by {}!
Tested on commit {}@{}. Tested on commit {}@{}.
Direct link to PR: <{}> Direct link to PR: <{}>
''').format(relevant_pr_number, slug, current_commit, relevant_pr_url) """).format(relevant_pr_number, slug, current_commit, relevant_pr_url)
anything_changed = False anything_changed = False
for status in latest: for status in latest:
tool = status['tool'] tool = status["tool"]
changed = False changed = False
create_issue_for_status = None # set to the status that caused the issue create_issue_for_status = None # set to the status that caused the issue
@@ -173,57 +182,70 @@ def update_latest(
old = status[os_] old = status[os_]
new = s.get(tool, old) new = s.get(tool, old)
status[os_] = new status[os_] = new
maintainers = ' '.join('@'+name for name in MAINTAINERS.get(tool, ())) maintainers = " ".join("@" + name for name in MAINTAINERS.get(tool, ()))
# comparing the strings, but they are ordered appropriately: # comparing the strings, but they are ordered appropriately:
# "test-pass" > "test-fail" > "build-fail" # "test-pass" > "test-fail" > "build-fail"
if new > old: if new > old:
# things got fixed or at least the status quo improved # things got fixed or at least the status quo improved
changed = True changed = True
message += '🎉 {} on {}: {} → {} (cc {}).\n' \ message += "🎉 {} on {}: {} → {} (cc {}).\n".format(
.format(tool, os_, old, new, maintainers) tool, os_, old, new, maintainers
)
elif new < old: elif new < old:
# tests or builds are failing and were not failing before # tests or builds are failing and were not failing before
changed = True changed = True
title = '💔 {} on {}: {} → {}' \ title = "💔 {} on {}: {} → {}".format(tool, os_, old, new)
.format(tool, os_, old, new) message += "{} (cc {}).\n".format(title, maintainers)
message += '{} (cc {}).\n' \
.format(title, maintainers)
# See if we need to create an issue. # See if we need to create an issue.
# Create issue if things no longer build. # Create issue if things no longer build.
# (No issue for mere test failures to avoid spurious issues.) # (No issue for mere test failures to avoid spurious issues.)
if new == 'build-fail': if new == "build-fail":
create_issue_for_status = new create_issue_for_status = new
if create_issue_for_status is not None: if create_issue_for_status is not None:
try: try:
issue( issue(
tool, create_issue_for_status, MAINTAINERS.get(tool, ()), tool,
relevant_pr_number, relevant_pr_user, LABELS.get(tool, []), create_issue_for_status,
MAINTAINERS.get(tool, ()),
relevant_pr_number,
relevant_pr_user,
LABELS.get(tool, []),
github_token, github_token,
) )
except HTTPError as e: except HTTPError as e:
# network errors will simply end up not creating an issue, but that's better # network errors will simply end up not creating an issue, but that's better
# than failing the entire build job # than failing the entire build job
print("HTTPError when creating issue for status regression: {0}\n{1!r}" print(
.format(e, e.read())) "HTTPError when creating issue for status regression: {0}\n{1!r}".format(
e, e.read()
)
)
except IOError as e: except IOError as e:
print("I/O error when creating issue for status regression: {0}".format(e)) print(
"I/O error when creating issue for status regression: {0}".format(
e
)
)
except: except:
print("Unexpected error when creating issue for status regression: {0}" print(
.format(sys.exc_info()[0])) "Unexpected error when creating issue for status regression: {0}".format(
sys.exc_info()[0]
)
)
raise raise
if changed: if changed:
status['commit'] = current_commit status["commit"] = current_commit
status['datetime'] = current_datetime status["datetime"] = current_datetime
anything_changed = True anything_changed = True
if not anything_changed: if not anything_changed:
return '' return ""
f.seek(0) f.seek(0)
f.truncate(0) f.truncate(0)
json.dump(latest, f, indent=4, separators=(',', ': ')) json.dump(latest, f, indent=4, separators=(",", ": "))
return message return message
@@ -231,12 +253,12 @@ def update_latest(
# There are variables declared within that are implicitly global; it is unknown # There are variables declared within that are implicitly global; it is unknown
# which ones precisely but at least this is true for `github_token`. # which ones precisely but at least this is true for `github_token`.
try: try:
if __name__ != '__main__': if __name__ != "__main__":
exit(0) exit(0)
cur_commit = sys.argv[1] cur_commit = sys.argv[1]
cur_datetime = datetime.datetime.now(datetime.timezone.utc).strftime( cur_datetime = datetime.datetime.now(datetime.timezone.utc).strftime(
'%Y-%m-%dT%H:%M:%SZ' "%Y-%m-%dT%H:%M:%SZ"
) )
cur_commit_msg = sys.argv[2] cur_commit_msg = sys.argv[2]
save_message_to_path = sys.argv[3] save_message_to_path = sys.argv[3]
@@ -244,21 +266,21 @@ try:
# assume that PR authors are also owners of the repo where the branch lives # assume that PR authors are also owners of the repo where the branch lives
relevant_pr_match = re.search( relevant_pr_match = re.search(
r'Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)', r"Auto merge of #([0-9]+) - ([^:]+):[^,]+, r=(\S+)",
cur_commit_msg, cur_commit_msg,
) )
if relevant_pr_match: if relevant_pr_match:
number = relevant_pr_match.group(1) number = relevant_pr_match.group(1)
relevant_pr_user = relevant_pr_match.group(2) relevant_pr_user = relevant_pr_match.group(2)
relevant_pr_number = 'rust-lang/rust#' + number relevant_pr_number = "rust-lang/rust#" + number
relevant_pr_url = 'https://github.com/rust-lang/rust/pull/' + number relevant_pr_url = "https://github.com/rust-lang/rust/pull/" + number
pr_reviewer = relevant_pr_match.group(3) pr_reviewer = relevant_pr_match.group(3)
else: else:
number = '-1' number = "-1"
relevant_pr_user = 'ghost' relevant_pr_user = "ghost"
relevant_pr_number = '<unknown PR>' relevant_pr_number = "<unknown PR>"
relevant_pr_url = '<unknown>' relevant_pr_url = "<unknown>"
pr_reviewer = 'ghost' pr_reviewer = "ghost"
message = update_latest( message = update_latest(
cur_commit, cur_commit,
@@ -270,28 +292,30 @@ try:
github_token, github_token,
) )
if not message: if not message:
print('<Nothing changed>') print("<Nothing changed>")
sys.exit(0) sys.exit(0)
print(message) print(message)
if not github_token: if not github_token:
print('Dry run only, not committing anything') print("Dry run only, not committing anything")
sys.exit(0) sys.exit(0)
with open(save_message_to_path, 'w') as f: with open(save_message_to_path, "w") as f:
f.write(message) f.write(message)
# Write the toolstate comment on the PR as well. # Write the toolstate comment on the PR as well.
issue_url = gh_url() + '/{}/comments'.format(number) issue_url = gh_url() + "/{}/comments".format(number)
response = urllib2.urlopen(urllib2.Request( response = urllib2.urlopen(
issue_url, urllib2.Request(
json.dumps({'body': maybe_remove_mention(message)}).encode(), issue_url,
{ json.dumps({"body": maybe_remove_mention(message)}).encode(),
'Authorization': 'token ' + github_token, {
'Content-Type': 'application/json', "Authorization": "token " + github_token,
} "Content-Type": "application/json",
)) },
)
)
response.read() response.read()
except HTTPError as e: except HTTPError as e:
print("HTTPError: %s\n%r" % (e, e.read())) print("HTTPError: %s\n%r" % (e, e.read()))
tests/debuginfo/auxiliary/dependency-with-embedded-visualizers.py
+3
View File
@@ -1,5 +1,6 @@
import gdb import gdb
class PersonPrinter: class PersonPrinter:
"Print a Person" "Print a Person"
@@ -11,6 +12,7 @@ class PersonPrinter:
def to_string(self): def to_string(self):
return "{} is {} years old.".format(self.name, self.age) return "{} is {} years old.".format(self.name, self.age)
def lookup(val): def lookup(val):
lookup_tag = val.type.tag lookup_tag = val.type.tag
if lookup_tag is None: if lookup_tag is None:
@@ -20,4 +22,5 @@ def lookup(val):
return None return None
gdb.current_objfile().pretty_printers.append(lookup) gdb.current_objfile().pretty_printers.append(lookup)
tests/debuginfo/embedded-visualizer-point.py
+3
View File
@@ -1,5 +1,6 @@
import gdb import gdb
class PointPrinter: class PointPrinter:
"Print a Point" "Print a Point"
@@ -11,6 +12,7 @@ class PointPrinter:
def to_string(self): def to_string(self):
return "({}, {})".format(self.x, self.y) return "({}, {})".format(self.x, self.y)
def lookup(val): def lookup(val):
lookup_tag = val.type.tag lookup_tag = val.type.tag
if lookup_tag is None: if lookup_tag is None:
@@ -20,4 +22,5 @@ def lookup(val):
return None return None
gdb.current_objfile().pretty_printers.append(lookup) gdb.current_objfile().pretty_printers.append(lookup)
tests/debuginfo/embedded-visualizer.py
+3
View File
@@ -1,5 +1,6 @@
import gdb import gdb
class LinePrinter: class LinePrinter:
"Print a Line" "Print a Line"
@@ -11,6 +12,7 @@ class LinePrinter:
def to_string(self): def to_string(self):
return "({}, {})".format(self.a, self.b) return "({}, {})".format(self.a, self.b)
def lookup(val): def lookup(val):
lookup_tag = val.type.tag lookup_tag = val.type.tag
if lookup_tag is None: if lookup_tag is None:
@@ -20,4 +22,5 @@ def lookup(val):
return None return None
gdb.current_objfile().pretty_printers.append(lookup) gdb.current_objfile().pretty_printers.append(lookup)
x.py
+6 -3
View File
@@ -6,7 +6,7 @@
# Parts of `bootstrap.py` use the `multiprocessing` module, so this entry point # Parts of `bootstrap.py` use the `multiprocessing` module, so this entry point
# must use the normal `if __name__ == '__main__':` convention to avoid problems. # must use the normal `if __name__ == '__main__':` convention to avoid problems.
if __name__ == '__main__': if __name__ == "__main__":
import os import os
import sys import sys
import warnings import warnings
@@ -32,14 +32,16 @@ if __name__ == '__main__':
# soft deprecation of old python versions # soft deprecation of old python versions
skip_check = os.environ.get("RUST_IGNORE_OLD_PYTHON") == "1" skip_check = os.environ.get("RUST_IGNORE_OLD_PYTHON") == "1"
if not skip_check and (major < 3 or (major == 3 and minor < 6)): if not skip_check and (major < 3 or (major == 3 and minor < 6)):
msg = cleandoc(""" msg = cleandoc(
"""
Using python {}.{} but >= 3.6 is recommended. Your python version Using python {}.{} but >= 3.6 is recommended. Your python version
should continue to work for the near future, but this will should continue to work for the near future, but this will
eventually change. If python >= 3.6 is not available on your system, eventually change. If python >= 3.6 is not available on your system,
please file an issue to help us understand timelines. please file an issue to help us understand timelines.
This message can be suppressed by setting `RUST_IGNORE_OLD_PYTHON=1` This message can be suppressed by setting `RUST_IGNORE_OLD_PYTHON=1`
""".format(major, minor)) """.format(major, minor)
)
warnings.warn(msg, stacklevel=1) warnings.warn(msg, stacklevel=1)
rust_dir = os.path.dirname(os.path.abspath(__file__)) rust_dir = os.path.dirname(os.path.abspath(__file__))
@@ -47,4 +49,5 @@ if __name__ == '__main__':
sys.path.insert(0, os.path.join(rust_dir, "src", "bootstrap")) sys.path.insert(0, os.path.join(rust_dir, "src", "bootstrap"))
import bootstrap import bootstrap
bootstrap.main() bootstrap.main()