Source code
Revision control
Copy as Markdown
Other Tools
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
import asyncio
import logging
import os
import re
from shutil import which
from mach.decorators import (
Command,
CommandArgument,
)
_log = logging.getLogger(__name__)
NO_RIPGREP_MSG = """
Could not find ripgrep. Perhaps it needs installing via your package manager
or `cargo install`.
""".strip()
RIPGREP_ERROR_MSG = """
An error occurred running ripgrep. Please check the following error messages:
{}
""".strip()
NO_MODULES_TO_REWRITE_MSG = """
Found no EXTRA_JS_MODULES we could convert in the moz.build file(s) passed.
""".strip()
CANNOT_CONVERT_ERROR_MSG = """
Unfortunately {} cannot be automatically converted.
The script isn't clever enough to convert everything else while leaving that
module alone. Consider (temporarily) removing this module from the moz.build
file, then running the script, and then re-adding the module into the
moz.build file in a copy of the original EXTRA_JS_MODULES instruction.
The list of excluded module patterns is in tools/use-moz-src/mach_commands.py
and should have a comment explaining why it cannot be automatically moved to
moz-src:///.
""".strip()
excluded_from_convert_re = list(
map(
re.compile,
[
"BackgroundTask_.*\\.sys\\.mjs", # Depend on this resource: path.
"BrowserUsageTelemetry\\.sys\\.mjs", # Browser + Android duplicates.
"BuiltInThemes\\.sys\\.mjs", # Thunderbird override
"CrashManager\\.sys.\\.mjs", # Preprocessed.
"ExtensionBrowsingData\\.sys\\.mjs", # Browser + Android duplicates.
"policies/schema\\.sys\\.mjs", # Thunderbird override
"Policies\\.sys\\.mjs", # Thunderbird override
"PromiseWorker\\.m?js", # Preprocessed.
"RFPTargetConstants\\.sys\\.mjs", # Preprocessed.
"ThemeVariableMap\\.sys\\.mjs", # Thunderbird override
],
)
)
def is_excluded_from_convert(path):
"""Returns true if the JSM file shouldn't be converted to ESM."""
path_str = str(path)
for expr in excluded_from_convert_re:
if expr.search(path_str):
return True
return False
def extract_info_from_mozbuild(command_context, paths):
mozbuilds_for_fixing = set()
urlmap = dict()
reader = command_context.mozbuild_reader(config_mode="empty")
for mozbuild_path in paths:
is_browser = mozbuild_path.startswith("browser") or (
mozbuild_path.startswith("devtools")
and not mozbuild_path.startswith("devtools/platform")
)
assignments = reader.find_variables_from_ast(
variables="EXTRA_JS_MODULES", path=mozbuild_path, all_relevant_files=False
)
for path, _variable, key, value in assignments:
module_path = path.replace("moz.build", "") + value
if is_excluded_from_convert(module_path):
_log.log(logging.ERROR, CANNOT_CONVERT_ERROR_MSG.format(value))
return [], dict()
newurl = "moz-src:///" + module_path
module_name = os.path.basename(module_path)
keystr = "/".join(key.split(".")) + "/" if key else ""
resource_suffix = "modules/" + keystr + module_name
if is_browser:
else:
mozbuilds_for_fixing.add(mozbuild_path)
return mozbuilds_for_fixing, urlmap
extra_js_modules_re = re.compile('EXTRA_JS_MODULES(["\\.\\w/\\[\\]]*) [+=]+')
def rewrite_mozbuilds(mozbuilds):
for path in mozbuilds:
with open(path, "r+", encoding="utf-8", newline="\n") as f:
contents = f.read()
contents = re.sub(extra_js_modules_re, "MOZ_SRC_FILES +=", contents)
f.seek(0)
f.write(contents)
f.truncate()
def _get_re_for_urls(urlmap):
old_urls = list(urlmap.keys())
prefix = os.path.commonprefix(old_urls)
prefixlen = len(prefix)
old_urls_without_prefix = map(lambda u: u[prefixlen:], old_urls)
escaped_prefix = re.escape(prefix)
# Put the "app" option in if needed:
suffixes = "|".join(map(re.escape, old_urls_without_prefix))
return escaped_prefix + "(" + suffixes + ")"
def handle_matching_file(file, urlmap_re, replace_fn):
with open(file, "r+", encoding="utf-8", newline="\n") as f:
contents = f.read()
f.seek(0)
contents = re.sub(urlmap_re, replace_fn, contents)
f.write(contents)
f.truncate()
async def find_and_replace_refs(urlmap):
urlmap_re = _get_re_for_urls(urlmap)
encoding = "utf-8"
cmd_args = [which("rg"), "-l", "-e", urlmap_re]
proc = await asyncio.subprocess.create_subprocess_exec(
*cmd_args, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
)
pending_file_writes = set()
affected_files = set()
def replace_fn(matchobj):
old_url = matchobj.group(0)
# Replace with the mapped URL or with itself if we can't find it.
return urlmap.get(old_url, old_url)
async with asyncio.TaskGroup() as tg:
async def handle_matching_files() -> None:
assert proc.stdout is not None
while matching_file := await proc.stdout.readline():
matching_file = matching_file.decode(encoding, "replace")
if not matching_file.endswith("\n"):
continue
matching_file = matching_file.strip()
if not matching_file:
continue
_log.log(logging.INFO, f"Rewriting {matching_file}")
affected_files.add(matching_file)
task = tg.create_task(
asyncio.to_thread(
handle_matching_file, matching_file, urlmap_re, replace_fn
)
)
pending_file_writes.add(task)
task.add_done_callback(pending_file_writes.discard)
async def handle_errors() -> None:
assert proc.stderr is not None
while chunk := await proc.stderr.read(10000):
_log.log(
logging.ERROR,
RIPGREP_ERROR_MSG.format(chunk.decode("utf-8", "replace")),
)
# We assume that if anything gets written to stderr by ripgrep,
# it'll also exit shortly and we don't need to deal with that
# separately by raising exceptions or whatever.
await asyncio.gather(handle_matching_files(), handle_errors(), proc.wait())
return affected_files
@Command(
"use-moz-src",
category="misc",
description="Convert use of EXTRA_JS_MODULES to MOZ_SRC_FILES",
)
@CommandArgument(
"paths",
nargs="+",
help="Path to the moz.build file(s) to convert.",
)
def use_moz_src(command_context, paths):
"""
This command does two things:
1. replace use of EXTRA_JS_MODULES in the moz.build file passed with
MOZ_SRC_FILES.
2. fix up consumers across the tree that rely on any files in
EXTRA_JS_MODULES using `resource` URLs to use `moz-src` ones
instead.
Note that this only uses the moz.build file in question; it will not
recurse into `DIRS`. If you want to convert subdirectories, use
e.g. `find` to collect the moz.build files and pass all of them on
the commandline.
Example:
./mach use-moz-src browser/components/moz.build
Recursive example:
find browser/components -iname 'moz.build' | xargs ./mach use-moz-src
"""
if not which("rg"):
_log.log(logging.ERROR, NO_RIPGREP_MSG)
return
mozbuilds_for_fixing, urlmap = extract_info_from_mozbuild(command_context, paths)
if not mozbuilds_for_fixing:
_log.log(logging.INFO, NO_MODULES_TO_REWRITE_MSG)
return
_log.log(
logging.INFO,
"Searching the tree and updating all references to:\n {}".format(
"\n ".join(urlmap.keys())
),
)
updated_files = asyncio.run(find_and_replace_refs(urlmap))
_log.log(
logging.INFO,
"Updating build files:\n {}".format("\n ".join(mozbuilds_for_fixing)),
)
rewrite_mozbuilds(mozbuilds_for_fixing)
if len(updated_files) > 0:
_log.log(logging.INFO, f"Formatting {len(updated_files)} affected files...")
command_context._mach_context.commands.dispatch(
"lint",
command_context._mach_context,
linters=["eslint"],
paths=updated_files,
fix=True,
)
_log.log(
logging.INFO, "Done. Make sure to test the result before submitting patches."
)