import changes from `dev' branch of rmottola/Arctic-Fox:

- Bug 1204192 - IonMonkey: MIPS: Split shareable code to mips-shared in BaselineIC-mips32. r=nbp (eeae38da4e)
- Bug 1204193 - IonMonkey: MIPS: Split shareable code to mips-shared in Bailouts-mips32. r=nbp (f24f496235)
- Bug 1204194 - IonMonkey: MIPS: Split shareable code to mips-shared in MoveEmitter-mips32. r=nbp (5c3bdc2fc2)
- Bug 1204214 - IonMonkey: MIPS: Split shareable code to mips-shared in BaselineCompiler-mips32. r=nbp (01903406d1)
- Bug 1205229 - IonMonkey: MIPS32: Make more CodeGenerator functions can be shared. r=nbp (d437fe618e)
- Bug 1209528 - IonMonkey: MIPS32: Add suffix 'f' for constant float32. r=arai (0f6600c083)
- Bug 1205232 - IonMonkey: MIPS32: Fix rounding of big negative float32 values in Ion. r=bbouvier (4652bb9b7a)
- Bug 1203044 - IonMonkey: MIPS32: Atomics operations should throw on oob access. r=lth (e898b3af95)
- Bug 1205135 - IonMonkey: MIPS: Split shareable code to mips-shared in CodeGenerator-mips32. r=nbp (b6d81b922d)
- readd some missing bits (2aa8c9fafd)
- Bug 1167189: Remove unnecessary checks after infallible allocations. r=bholley (fa7d8b9f7b)
- Bug 1209398 - Make AC_SUBST_SET emit a list of unique items instead of an actual set. r=gps (83e5877882)
- Bug 904572 - Add support for generating clang compilation database; r=glandium,r=gps (ab3e6e62fa)
- Bug 1209398 - Enable the FasterMake backend by default for desktop Firefox builds. r=gps (29f6fb3883)
- Bug 1204719 - Don't create interfaces.manifest files from the build backend. r=gps (5a63e36259)
- Bug 786520 - Install things to $(DIST)/branding from moz.build instead of manual rules in Makefile.ins. r=mshal (652552b547)
- Bug 1160563 - Part 1: Make ANDROID_RES_DIRS a moz.build variable. r=gps (e0719524f8)
- Bug 1159390 - Set sharedUserId in robocop.apk. r=gbrown (dde6716e2d)
- Bug 1160030 - Use TEST_HARNESS_FILES to install Robocop test files. r=froydnj (00cd01c3f2)
- Bug 1160030 - Use TEST_HARNESS_FILES to install Robocop ini files. r=froydnj (66d52518e7)
- Bug 938659 - Part 2: build system changes. r=mfinkle (0833d22ba3)
- Bug 1160563 - Pre: Allow ANDROID_RES_DIRS that are not relative to srcdir. r=gps (0d8a3ce66e)
- Bug 1160563 - Part 2: Make ANDROID_ASSETS_DIRS a moz.build variable. r=gps (9f4c132e2a)
- Bug 1195388 - Part 1: Make ANDROID_APK_{NAME,PACKAGE} moz.build variables. r=gps (4a2fa3fc09)
- Bug 1195388 - Part 2: Add ANDROID_EXTRA_{PACKAGES,RES_DIRS} moz.buildvariables. r=gps (7ad49e355e)
- Bug 1207893 - Change how we track object consumption from the build backend. r=gps (20ccc5b67a)
- Bug 1184405 - Add annotations for tags, file patterns, and test flavos to moz.build to specify tests potentially impacted by source files. r=gps (2db17131cc)
- Bug 1171105 - Ability to aggregate metadata from Files instances; r=glandium (59c8759e61)
- Bug 1184405 - Add a container type to mozbuild with a namedtuple-likeinterface and typed, mutable fields. r=gps (3062dbae71)
- Bug 1138283 - Fix bad documentation around wildcard patterns; r=glandium (5785d3284d)
- Bug 1155816 - part 2 - move EXTRA_*COMPONENTS manifest check to buildbackend time; r=mshal (c539616b45)
- Bug 1161270 - Add source manifest to reftest manifest representation in moz.build.;r=gps,roc (376b080084)
- Bug 1180813 - Use a cache for BuildConfig to improve mach speed. r=gps (d431b1ec23)
- Bug 1207882 - Associate an install target with every ContextDerived object. r=gps (62f63487a7)
- Bug 1126228 - Directory (--directory) argument for |mach warnings-list| and |mach warnings-summary|. r=gps (37a85d904f)
- Bug 1210642 - Allow to pass defines overrides when processing install manifests. r=gps (5eb483dc8d)
- Bug 1210642 - Add support for --silence-missing-directive-warnings for preprocessing within install manifests. r=gps (3a07d30b36)
- Bug 1176642 - Use absolute_import in mozpack; r=glandium (863ab20c4a)
- Bug 1162826 - Properly display full moz.build processing errors when empty KeyError or ValueError are thrown. r=mshal (57e8367680)
- Bug 1184405 - Add a mach command to expose test-deps file info. r=gps (7bcc10679c)
- Bug 1195735, r=zer0 (7fe14224e3)
- Bug 1205166 - IonMonkey: MIPS: Import MIPS64 support into Architecture-mips-shared. r=nbp f=rankov (84e1f9ec5d)
- Bug 1205166 - IonMonkey: MIPS64: Import Architecture-mips64. r=nbp f=rankov (fe173dd5af)
- Bug 1210322 - IonMonkey: MIPS: Rename BaseFloatRegister/s to FloatRegister/sMIPSShared. r=nbp (ac262cbb5d)
- Bug 1205167 - IonMonkey: MIPS64: Import Assembler-mips64. r=nbp f=rankov (079e84ea49)
- Bug 1215555 - Improve simulated OOM testing of ARM assembler buffer and fix bugs r=jolesen (253b976cec)
- Bug 1205167 - IonMonkey: MIPS: Import MIPS64 support into Assembler-mips-shared. r=froydnj f=rankov (94b23ac585)
- Bug 1213146 - IonMonkey: MIPS: Fix build failure caused by bug 1194139. r=nbp (12f7d5d572)
- Bug 1205566 - IonMonkey: MIPS: bailoutCmp32/Ptr optimization. r=nbp (9a40ee80d0)
- Bug 1209873 - IonMonkey; MIPS: Implement memory barrier. r=lth (964535cb6b)
- Bug 1209572 - IonMonkey: MIPS32: Goto done by short jump in convertUInt32ToFloat32. r=nbp (0b388199e7)
- pointer style (312fc6fe4f)
- missing bit of Bug 1140890 - Make sure the first argument cannot bail in between negative zero removal and creating result in substraction, r=nbp (afcc9bfdc7)
- and sync more build scripts with frontend branch

python/mach_commands.py

@@ -5,10 +5,13 @@
 from __future__ import absolute_import, print_function, unicode_literals
 
 import argparse
-import glob
 import logging
 import mozpack.path as mozpath
 import os
+import platform
+import subprocess
+import sys
+import which
 
 from mozbuild.base import (
     MachCommandBase,
@@ -20,6 +23,46 @@ from mach.decorators import (
     Command,
 )
 
+ESLINT_NOT_FOUND_MESSAGE = '''
+Could not find eslint!  We looked at the --binary option, at the ESLINT
+environment variable, and then at your path.  Install eslint and needed plugins
+with
+
+mach eslint --setup
+
+and try again.
+'''.strip()
+
+NODE_NOT_FOUND_MESSAGE = '''
+nodejs is either not installed or is installed to a non-standard path.
+Please install nodejs from https://nodejs.org and try again.
+
+Valid installation paths:
+'''.strip()
+
+NPM_NOT_FOUND_MESSAGE = '''
+Node Package Manager (npm) is either not installed or installed to a
+non-standard path. Please install npm from https://nodejs.org (it comes as an
+option in the node installation) and try again.
+
+Valid installation paths:
+'''.strip()
+
+ESLINT_PROMPT = '''
+Would you like to use eslint
+'''.strip()
+
+ESLINT_PLUGIN_MOZILLA_PROMPT = '''
+eslint-plugin-mozilla is an eslint plugin containing rules that help enforce
+JavaScript coding standards in the Mozilla project. Would you like to use this
+plugin
+'''.strip()
+
+ESLINT_PLUGIN_REACT_PROMPT = '''
+eslint-plugin-react is an eslint plugin containing rules that help React
+developers follow strict guidelines. Would you like to install it
+'''.strip()
+
 
 @CommandProvider
 class MachCommands(MachCommandBase):
@@ -53,6 +96,7 @@ class MachCommands(MachCommandBase):
         help='Tests to run. Each test can be a single file or a directory.')
     def python_test(self, tests, verbose=False, stop=False):
         self._activate_virtualenv()
+        import glob
 
         # Python's unittest, and in particular discover, has problems with
         # clashing namespaces when importing multiple test modules. What follows
@@ -119,3 +163,157 @@ class MachCommands(MachCommandBase):
                 return 1
 
         return 0 if return_code == 0 else 1
+
+    @Command('eslint', category='devenv',
+        description='Run eslint or help configure eslint for optimal development.')
+    @CommandArgument('-s', '--setup', default=False, action='store_true',
+        help='configure eslint for optimal development.')
+    @CommandArgument('path', nargs='?', default='.',
+        help='Path to files to lint, like "browser/components/loop" '
+            'or "mobile/android". '
+            'Defaults to the current directory if not given.')
+    @CommandArgument('-e', '--ext', default='[.js,.jsm,.jsx]',
+        help='Filename extensions to lint, default: "[.js,.jsm,.jsx]".')
+    @CommandArgument('-b', '--binary', default=None,
+        help='Path to eslint binary.')
+    @CommandArgument('args', nargs=argparse.REMAINDER)  # Passed through to eslint.
+    def eslint(self, setup, path, ext=None, binary=None, args=[]):
+        '''Run eslint.'''
+
+        if setup:
+            return self.eslint_setup()
+
+        if not binary:
+            binary = os.environ.get('ESLINT', None)
+            if not binary:
+                try:
+                    binary = which.which('eslint')
+                except which.WhichError:
+                    pass
+
+        if not binary:
+            print(ESLINT_NOT_FOUND_MESSAGE)
+            return 1
+
+        # The cwd below is unfortunate.  eslint --config=PATH/TO/.eslintrc works,
+        # but --ignore-path=PATH/TO/.eslintignore treats paths as relative to
+        # the current directory, rather than as relative to the location of
+        # .eslintignore (see https://github.com/eslint/eslint/issues/1382).
+        # mach commands always execute in the topsrcdir, so we could make all
+        # paths in .eslint relative to the topsrcdir, but it's not clear if
+        # that's a good choice for future eslint and IDE integrations.
+        # Unfortunately, running after chdir does not print the full path to
+        # files (convenient for opening with copy-and-paste).  In the meantime,
+        # we just print the active path.
+
+        self.log(logging.INFO, 'eslint', {'binary': binary, 'path': path},
+            'Running {binary} in {path}')
+
+        cmd_args = [binary,
+            '--ext', ext,  # This keeps ext as a single argument.
+        ] + args
+        # Path must come after arguments.  Path is '.' due to cwd below.
+        cmd_args += ['.']
+
+        return self.run_process(cmd_args,
+            cwd=path,
+            pass_thru=True,  # Allow user to run eslint interactively.
+            ensure_exit_code=False,  # Don't throw on non-zero exit code.
+        )
+    def eslint_setup(self, update_only=False):
+        """Ensure eslint is optimally configured.
+
+        This command will inspect your eslint configuration and
+        guide you through an interactive wizard helping you configure
+        eslint for optimal use on Mozilla projects.
+        """
+        sys.path.append(os.path.dirname(__file__))
+
+        # At the very least we need node installed.
+        nodePath = self.getNodeOrNpmPath("node")
+        if not nodePath:
+            return 1
+
+        npmPath = self.getNodeOrNpmPath("npm")
+        if not npmPath:
+            return 1
+
+        # Install eslint.
+        print("Installing eslint...")
+        with open(os.devnull, "w") as fnull:
+            subprocess.call([npmPath, "install", "eslint", "-g"],
+                            stdout=fnull, stderr=fnull)
+
+        # Install eslint-plugin-mozilla.
+        print("")
+        print("Installing eslint-plugin-mozilla...")
+        with open(os.devnull, "w") as fnull:
+            subprocess.call([npmPath, "link"],
+                            cwd="testing/eslint-plugin-mozilla",
+                            stdout=fnull, stderr=fnull)
+
+        # Install eslint-plugin-react.
+        print("")
+        print("Installing eslint-plugin-react...")
+        with open(os.devnull, "w") as fnull:
+            subprocess.call([npmPath, "install", "-g", "eslint-plugin-react"],
+                            stdout=fnull, stderr=fnull)
+
+    def getPossibleNodePathsWin(self):
+        """
+        Return possible nodejs paths on Windows.
+        """
+        if platform.system() != "Windows":
+            return []
+
+        return {
+            "%s\\nodejs" % os.environ.get("SystemDrive"),
+            os.path.join(os.environ.get("ProgramFiles"), "nodejs"),
+            os.path.join(os.environ.get("PROGRAMW6432"), "nodejs"),
+            os.path.join(os.environ.get("PROGRAMFILES"), "nodejs")
+        }
+
+    def getNodeOrNpmPath(self, filename):
+        """
+        Return the nodejs or npm path.
+        """
+        try:
+            appPath = which.which(filename)
+            return appPath
+        except which.WhichError:
+            pass
+
+        if platform.system() == "Windows":
+            try:
+                for ext in ["", ".cmd", ".exe"]:
+                    nodeOrNpmPath = which.which(filename + ext,
+                                                path=self.getPossibleNodePathsWin())
+                    if self.is_valid(nodeOrNpmPath):
+                        return nodeOrNpmPath
+            except which.WhichError:
+                pass
+
+        if filename == "node":
+            print(NODE_NOT_FOUND_MESSAGE)
+        elif filename == "npm":
+            print(NPM_NOT_FOUND_MESSAGE)
+
+        if platform.system() == "Windows":
+            appPaths = self.getPossibleNodePathsWin()
+
+            for p in appPaths:
+                print("  - " + p)
+        elif platform.system() == "Darwin":
+            print("  - /usr/local/bin/node")
+        elif platform.system() == "Linux":
+            print("  - /usr/bin/nodejs")
+
+        return None
+
+    def is_valid(self, path):
+        try:
+            with open(os.devnull, "w") as fnull:
+                subprocess.check_call([path, "--version"], stdout=fnull)
+                return True
+        except subprocess.CalledProcessError:
+            return False

python/mozbuild/mozbuild/action/buildlist.py

@@ -12,12 +12,16 @@ from __future__ import absolute_import, print_function
 import sys
 import os
 
-from mozbuild.util import lock_file
+from mozbuild.util import (
+    ensureParentDir,
+    lock_file,
+)
 
 def addEntriesToListFile(listFile, entries):
   """Given a file |listFile| containing one entry per line,
   add each entry in |entries| to the file, unless it is already
   present."""
+  ensureParentDir(listFile)
   lock = lock_file(listFile + ".lck")
   try:
     if os.path.exists(listFile):

python/mozbuild/mozbuild/action/process_install_manifest.py

@@ -8,6 +8,7 @@ import argparse
 import sys
 from mozpack.copier import FileCopier
 from mozpack.manifests import InstallManifest
+from mozbuild.util import DefinesAction
 
 
 COMPLETE = 'From {dest}: Kept {existing} existing; Added/updated {updated}; ' \
@@ -17,19 +18,36 @@ COMPLETE = 'From {dest}: Kept {existing} existing; Added/updated {updated}; ' \
 def process_manifest(destdir, paths,
         remove_unaccounted=True,
         remove_all_directory_symlinks=True,
-        remove_empty_directories=True):
+        remove_empty_directories=True,
+        defines={}):
     manifest = InstallManifest()
     for path in paths:
         manifest |= InstallManifest(path=path)
 
     copier = FileCopier()
-    manifest.populate_registry(copier)
+    manifest.populate_registry(copier, defines_override=defines)
     return copier.copy(destdir,
         remove_unaccounted=remove_unaccounted,
         remove_all_directory_symlinks=remove_all_directory_symlinks,
         remove_empty_directories=remove_empty_directories)
 
 
+class DefinesAction(argparse.Action):
+    def __call__(self, parser, namespace, values, option_string):
+        defines = getattr(namespace, self.dest)
+        if defines is None:
+            defines = {}
+        values = values.split('=', 1)
+        if len(values) == 1:
+            name, value = values[0], 1
+        else:
+            name, value = values
+            if value.isdigit():
+                value = int(value)
+        defines[name] = value
+        setattr(namespace, self.dest, defines)
+
+
 def main(argv):
     parser = argparse.ArgumentParser(
         description='Process install manifest files.')
@@ -42,13 +60,17 @@ def main(argv):
         help='Do not remove all directory symlinks from destination.')
     parser.add_argument('--no-remove-empty-directories', action='store_true',
         help='Do not remove empty directories from destination.')
+    parser.add_argument('-D', action=DefinesAction,
+        dest='defines', metavar="VAR[=VAL]",
+        help='Define a variable to override what is specified in the manifest')
 
     args = parser.parse_args(argv)
 
     result = process_manifest(args.destdir, args.manifests,
         remove_unaccounted=not args.no_remove,
         remove_all_directory_symlinks=not args.no_remove_all_directory_symlinks,
-        remove_empty_directories=not args.no_remove_empty_directories)
+        remove_empty_directories=not args.no_remove_empty_directories,
+        defines=args.defines)
 
     print(COMPLETE.format(dest=args.destdir,
         existing=result.existing_files_count,

python/mozbuild/mozbuild/backend/android_eclipse.py

@@ -55,21 +55,19 @@ class AndroidEclipseBackend(CommonBackend):
         """Write out Android Eclipse project files."""
 
         if not isinstance(obj, ContextDerived):
-            return
+            return False
 
-        CommonBackend.consume_object(self, obj)
+        if CommonBackend.consume_object(self, obj):
+            # If CommonBackend acknowledged the object, we're done with it.
+            return True
 
-        # If CommonBackend acknowledged the object, we're done with it.
-        if obj._ack:
-            return
-
-        # We don't want to handle most things, so we just acknowledge all objects...
-        obj.ack()
-
-        # ... and handle the one case we care about specially.
+        # Handle the one case we care about specially.
         if isinstance(obj, ContextWrapped) and isinstance(obj.wrapped, AndroidEclipseProjectData):
             self._process_android_eclipse_project_data(obj.wrapped, obj.srcdir, obj.objdir)
 
+        # We don't want to handle most things, so we just acknowledge all objects
+        return True
+
     def consume_finished(self):
         """The common backend handles WebIDL and test files. We don't handle
         these, so we don't call our superclass.

python/mozbuild/mozbuild/backend/base.py

@@ -117,7 +117,8 @@ class BuildBackend(LoggingMixin):
         """
         for obj in objs:
             obj_start = time.time()
-            self.consume_object(obj)
+            if not self.consume_object(obj):
+                raise Exception('Unhandled object of type %s' % type(obj))
             self._execution_time += time.time() - obj_start
 
             if isinstance(obj, ContextDerived):

python/mozbuild/mozbuild/backend/common.py

@@ -206,7 +206,7 @@ class CommonBackend(BuildBackend):
             # Do not handle ConfigFileSubstitution for Makefiles. Leave that
             # to other
             if mozpath.basename(obj.output_path) == 'Makefile':
-                return
+                return False
             with self._get_preprocessor(obj) as pp:
                 pp.do_include(obj.input_path)
             self.backend_input_files.add(obj.input_path)
@@ -251,9 +251,9 @@ class CommonBackend(BuildBackend):
             if hasattr(self, '_process_unified_sources'):
                 self._process_unified_sources(obj)
         else:
-            return
+            return False
 
-        obj.ack()
+        return True
 
     def consume_finished(self):
         if len(self._idl_manager.idls):

python/mozbuild/mozbuild/backend/configenvironment.py

@@ -27,6 +27,8 @@ else:
 class BuildConfig(object):
     """Represents the output of configure."""
 
+    _CODE_CACHE = {}
+
     def __init__(self):
         self.topsrcdir = None
         self.topobjdir = None
@@ -35,26 +37,35 @@ class BuildConfig(object):
         self.substs = {}
         self.files = []
 
-    @staticmethod
-    def from_config_status(path):
+    @classmethod
+    def from_config_status(cls, path):
         """Create an instance from a config.status file."""
+        code_cache = cls._CODE_CACHE
+        mtime = os.path.getmtime(path)
 
-        with open(path, 'rt') as fh:
-            source = fh.read()
-            code = compile(source, path, 'exec', dont_inherit=1)
-            g = {
-                '__builtins__': __builtins__,
-                '__file__': path,
-            }
-            l = {}
-            exec(code, g, l)
+        # cache the compiled code as it can be reused
+        # we cache it the first time, or if the file changed
+        if not path in code_cache or code_cache[path][0] != mtime:
+            with open(path, 'rt') as fh:
+                source = fh.read()
+                code_cache[path] = (
+                    mtime,
+                    compile(source, path, 'exec', dont_inherit=1)
+                )
 
-            config = BuildConfig()
+        g = {
+            '__builtins__': __builtins__,
+            '__file__': path,
+        }
+        l = {}
+        exec(code_cache[path][1], g, l)
 
-            for name in l['__all__']:
-                setattr(config, name, l[name])
+        config = BuildConfig()
 
-            return config
+        for name in l['__all__']:
+            setattr(config, name, l[name])
+
+        return config
 
 
 class ConfigEnvironment(object):

python/mozbuild/mozbuild/backend/cpp_eclipse.py

@@ -65,8 +65,6 @@ class CppEclipseBackend(CommonBackend):
         return os.path.join(srcdir_parent, workspace_dirname)
 
     def consume_object(self, obj):
-        obj.ack()
-
         reldir = getattr(obj, 'relativedir', None)
 
         # Note that unlike VS, Eclipse' indexer seem to crawl the headers and
@@ -74,6 +72,8 @@ class CppEclipseBackend(CommonBackend):
         if isinstance(obj, Defines):
             self._paths_to_defines.setdefault(reldir, {}).update(obj.defines)
 
+        return True
+
     def consume_finished(self):
         settings_dir = os.path.join(self._project_dir, '.settings')
         launch_dir = os.path.join(self._project_dir, 'RunConfigurations')

python/mozbuild/mozbuild/backend/fastermake.py

@@ -0,0 +1,244 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+from __future__ import absolute_import, unicode_literals
+
+from mozbuild.backend.common import CommonBackend
+from mozbuild.frontend.data import (
+    ContextDerived,
+    Defines,
+    DistFiles,
+    FinalTargetFiles,
+    JARManifest,
+    JavaScriptModules,
+    JsPreferenceFile,
+    Resources,
+    VariablePassthru,
+)
+from mozbuild.makeutil import Makefile
+from mozbuild.util import OrderedDefaultDict
+from mozpack.manifests import InstallManifest
+import mozpack.path as mozpath
+from collections import OrderedDict
+from itertools import chain
+
+
+class FasterMakeBackend(CommonBackend):
+    def _init(self):
+        super(FasterMakeBackend, self)._init()
+
+        self._seen_directories = set()
+        self._defines = dict()
+        self._jar_manifests = OrderedDict()
+
+        self._manifest_entries = OrderedDefaultDict(list)
+
+        self._install_manifests = OrderedDefaultDict(InstallManifest)
+
+    def _add_preprocess(self, obj, path, dest, **kwargs):
+        target = mozpath.basename(path)
+        # This matches what PP_TARGETS do in config/rules.
+        if target.endswith('.in'):
+            target = target[:-3]
+        depfile = mozpath.join(
+            self.environment.topobjdir, 'faster', '.deps',
+            mozpath.join(obj.install_target, dest, target).replace('/', '_'))
+        self._install_manifests[obj.install_target].add_preprocess(
+            mozpath.join(obj.srcdir, path),
+            mozpath.join(dest, target),
+            depfile,
+            **kwargs)
+
+    def consume_object(self, obj):
+        if not isinstance(obj, Defines) and isinstance(obj, ContextDerived):
+            defines = self._defines.get(obj.objdir, {})
+            if defines:
+                defines = defines.defines
+
+        if isinstance(obj, Defines):
+            self._defines[obj.objdir] = obj
+
+            # We're assuming below that Defines come first for a given objdir,
+            # which is kind of set in stone from the order things are treated
+            # in emitter.py.
+            assert obj.objdir not in self._seen_directories
+
+        elif isinstance(obj, JARManifest) and \
+                obj.install_target.startswith('dist/bin'):
+            defines = self._defines.get(obj.objdir, [])
+            if defines:
+                defines = list(defines.get_defines())
+            self._jar_manifests[obj.path] = (obj.objdir,
+                                             obj.install_target,
+                                             defines)
+
+        elif isinstance(obj, VariablePassthru) and \
+                obj.install_target.startswith('dist/bin'):
+            for f in obj.variables.get('EXTRA_COMPONENTS', {}):
+                path = mozpath.join(obj.install_target, 'components',
+                                    mozpath.basename(f))
+                self._install_manifests[obj.install_target].add_symlink(
+                    mozpath.join(obj.srcdir, f),
+                    mozpath.join('components', mozpath.basename(f))
+                )
+                if f.endswith('.manifest'):
+                    manifest = mozpath.join(obj.install_target,
+                                            'chrome.manifest')
+                    self._manifest_entries[manifest].append(
+                        'manifest components/%s' % mozpath.basename(f))
+
+            for f in obj.variables.get('EXTRA_PP_COMPONENTS', {}):
+                self._add_preprocess(obj, f, 'components', defines=defines)
+
+                if f.endswith('.manifest'):
+                    manifest = mozpath.join(obj.install_target,
+                                            'chrome.manifest')
+                    self._manifest_entries[manifest].append(
+                        'manifest components/%s' % mozpath.basename(f))
+
+        elif isinstance(obj, JavaScriptModules) and \
+                obj.install_target.startswith('dist/bin'):
+            for path, strings in obj.modules.walk():
+                base = mozpath.join('modules', path)
+                for f in strings:
+                    if obj.flavor == 'extra':
+                        self._install_manifests[obj.install_target].add_symlink(
+                            mozpath.join(obj.srcdir, f),
+                            mozpath.join(base, mozpath.basename(f))
+                        )
+                    elif obj.flavor == 'extra_pp':
+                        self._add_preprocess(obj, f, base, defines=defines)
+
+        elif isinstance(obj, JsPreferenceFile) and \
+                obj.install_target.startswith('dist/bin'):
+            # The condition for the directory value in config/rules.mk is:
+            # ifneq (,$(DIST_SUBDIR)$(XPI_NAME)$(LIBXUL_SDK))
+            # - LIBXUL_SDK is not supported (it likely doesn't work in the
+            # recursive backend anyways
+            # - when XPI_NAME is set, obj.install_target will start with
+            # dist/xpi-stage
+            # - when DIST_SUBDIR is set, obj.install_target will start with
+            # dist/bin/$(DIST_SUBDIR)
+            # So an equivalent condition that is not cumbersome for us and that
+            # is enough at least for now is checking if obj.install_target is
+            # different from dist/bin.
+            if obj.install_target == 'dist/bin':
+                pref_dir = 'defaults/pref'
+            else:
+                pref_dir = 'defaults/preferences'
+
+            dest = mozpath.join(obj.install_target, pref_dir,
+                                mozpath.basename(obj.path))
+            # We preprocess these, but they don't necessarily have preprocessor
+            # directives, so tell the preprocessor to not complain about that.
+            self._add_preprocess(obj, obj.path, pref_dir, defines=defines,
+                                 silence_missing_directive_warnings=True)
+
+        elif isinstance(obj, Resources) and \
+                obj.install_target.startswith('dist/bin'):
+            for path, strings in obj.resources.walk():
+                base = mozpath.join('res', path)
+                for f in strings:
+                    flags = strings.flags_for(f)
+                    if flags and flags.preprocess:
+                        self._add_preprocess(obj, f, base, marker='%',
+                                             defines=obj.defines)
+                    else:
+                        self._install_manifests[obj.install_target].add_symlink(
+                            mozpath.join(obj.srcdir, f),
+                            mozpath.join(base, mozpath.basename(f))
+                        )
+
+        elif isinstance(obj, FinalTargetFiles) and \
+                obj.install_target.startswith('dist/bin'):
+            for path, strings in obj.files.walk():
+                base = mozpath.join(obj.install_target, path)
+                for f in strings:
+                    self._install_manifests[obj.install_target].add_symlink(
+                        mozpath.join(obj.srcdir, f),
+                        mozpath.join(path, mozpath.basename(f))
+                    )
+
+        elif isinstance(obj, DistFiles) and \
+                obj.install_target.startswith('dist/bin'):
+            # We preprocess these, but they don't necessarily have preprocessor
+            # directives, so tell the preprocessor to not complain about that.
+            for f in obj.files:
+                self._add_preprocess(obj, f, '', defines=defines,
+                                     silence_missing_directive_warnings=True)
+
+        else:
+            # We currently ignore a lot of object types, so just acknowledge
+            # everything.
+            return True
+
+        self._seen_directories.add(obj.objdir)
+        return True
+
+    def consume_finished(self):
+        mk = Makefile()
+        # Add the default rule at the very beginning.
+        mk.create_rule(['default'])
+        mk.add_statement('TOPSRCDIR = %s' % self.environment.topsrcdir)
+        mk.add_statement('TOPOBJDIR = %s' % self.environment.topobjdir)
+
+        # Add a few necessary variables inherited from configure
+        for var in (
+            'PYTHON',
+            'ACDEFINES',
+            'MOZ_CHROME_FILE_FORMAT',
+        ):
+            mk.add_statement('%s = %s' % (var, self.environment.substs[var]))
+
+        # Add all necessary information for jar manifest processing
+        jar_mn_targets = []
+
+        for path, (objdir, install_target, defines) in \
+                self._jar_manifests.iteritems():
+            rel_manifest = mozpath.relpath(path, self.environment.topsrcdir)
+            target = rel_manifest.replace('/', '-')
+            assert target not in jar_mn_targets
+            jar_mn_targets.append(target)
+            target = 'jar-%s' % target
+            mk.create_rule([target]).add_dependencies([path])
+            if objdir != mozpath.join(self.environment.topobjdir,
+                                      mozpath.dirname(rel_manifest)):
+                mk.create_rule([target]).add_dependencies(
+                    ['objdir = %s' % objdir])
+            if install_target != 'dist/bin':
+                mk.create_rule([target]).add_dependencies(
+                    ['install_target = %s' % install_target])
+            if defines:
+                mk.create_rule([target]).add_dependencies(
+                    ['defines = %s' % ' '.join(defines)])
+
+        mk.add_statement('JAR_MN_TARGETS = %s' % ' '.join(jar_mn_targets))
+
+        # Add information for chrome manifest generation
+        manifest_targets = []
+
+        for target, entries in self._manifest_entries.iteritems():
+            manifest_targets.append(target)
+            target = '$(TOPOBJDIR)/%s' % target
+            mk.create_rule([target]).add_dependencies(
+                ['content = %s' % ' '.join('"%s"' % e for e in entries)])
+
+        mk.add_statement('MANIFEST_TARGETS = %s' % ' '.join(manifest_targets))
+
+        # Add information for install manifests.
+        mk.add_statement('INSTALL_MANIFESTS = %s'
+                         % ' '.join(self._install_manifests.keys()))
+
+        mk.add_statement('include $(TOPSRCDIR)/config/faster/rules.mk')
+
+        for base, install_manifest in self._install_manifests.iteritems():
+            with self._write_file(
+                    mozpath.join(self.environment.topobjdir, 'faster',
+                                 'install_%s' % base.replace('/', '_'))) as fh:
+                install_manifest.write(fileobj=fh)
+
+        with self._write_file(
+                mozpath.join(self.environment.topobjdir, 'faster',
+                             'Makefile')) as fh:
+            mk.dump(fh, removal_guard=False)

python/mozbuild/mozbuild/backend/recursivemake.py

@@ -26,6 +26,10 @@ import mozpack.path as mozpath
 
 from .common import CommonBackend
 from ..frontend.data import (
+    AndroidAssetsDirs,
+    AndroidResDirs,
+    AndroidExtraResDirs,
+    AndroidExtraPackages,
     AndroidEclipseProjectData,
     BrandingFiles,
     ConfigFileSubstitution,
@@ -71,6 +75,11 @@ from ..util import (
 from ..makeutil import Makefile
 
 MOZBUILD_VARIABLES = [
+    b'ANDROID_APK_NAME',
+    b'ANDROID_APK_PACKAGE',
+    b'ANDROID_ASSETS_DIRS',
+    b'ANDROID_EXTRA_PACKAGES',
+    b'ANDROID_EXTRA_RES_DIRS',
     b'ANDROID_GENERATED_RESFILES',
     b'ANDROID_RES_DIRS',
     b'ASFLAGS',
@@ -419,11 +428,11 @@ class RecursiveMakeBackend(CommonBackend):
         """Write out build files necessary to build with recursive make."""
 
         if not isinstance(obj, ContextDerived):
-            return
+            return False
 
         backend_file = self._get_backend_file_for(obj)
 
-        CommonBackend.consume_object(self, obj)
+        consumed = CommonBackend.consume_object(self, obj)
 
         # CommonBackend handles XPIDLFile and TestManifest, but we want to do
         # some extra things for them.
@@ -436,8 +445,8 @@ class RecursiveMakeBackend(CommonBackend):
             self._process_test_manifest(obj, backend_file)
 
         # If CommonBackend acknowledged the object, we're done with it.
-        if obj._ack:
-            return
+        if consumed:
+            return True
 
         if isinstance(obj, DirectoryTraversal):
             self._process_directory_traversal(obj, backend_file)
@@ -514,7 +523,7 @@ class RecursiveMakeBackend(CommonBackend):
 
         elif isinstance(obj, Resources):
             self._process_resources(obj, obj.resources, backend_file)
-        
+
         elif isinstance(obj, BrandingFiles):
             self._process_branding_files(obj, obj.files, backend_file)
 
@@ -569,7 +578,7 @@ class RecursiveMakeBackend(CommonBackend):
             elif isinstance(obj.wrapped, AndroidEclipseProjectData):
                 self._process_android_eclipse_project_data(obj.wrapped, backend_file)
             else:
-                return
+                return False
 
         elif isinstance(obj, SharedLibrary):
             self._process_shared_library(obj, backend_file)
@@ -594,9 +603,26 @@ class RecursiveMakeBackend(CommonBackend):
             for f in obj.files:
                 backend_file.write('DIST_FILES += %s\n' % f)
 
+        elif isinstance(obj, AndroidResDirs):
+            for p in obj.paths:
+                backend_file.write('ANDROID_RES_DIRS += %s\n' % p.full_path)
+
+        elif isinstance(obj, AndroidAssetsDirs):
+            for p in obj.paths:
+                backend_file.write('ANDROID_ASSETS_DIRS += %s\n' % p.full_path)
+
+        elif isinstance(obj, AndroidExtraResDirs):
+            for p in obj.paths:
+                backend_file.write('ANDROID_EXTRA_RES_DIRS += %s\n' % p.full_path)
+
+        elif isinstance(obj, AndroidExtraPackages):
+            for p in obj.packages:
+                backend_file.write('ANDROID_EXTRA_PACKAGES += %s\n' % p)
+
         else:
-            return
-        obj.ack()
+            return False
+
+        return True
 
     def _fill_root_mk(self):
         """
@@ -1066,6 +1092,7 @@ INSTALL_TARGETS += %(prefix)s
         modules = manager.modules
         xpt_modules = sorted(modules.keys())
         xpt_files = set()
+        registered_xpt_files = set()
 
         mk = Makefile()
 
@@ -1097,15 +1124,16 @@ INSTALL_TARGETS += %(prefix)s
         rules = StringIO()
         mk.dump(rules, removal_guard=False)
 
-        # Write out manifests defining interfaces
+        interfaces_manifests = []
         dist_dir = mozpath.join(self.environment.topobjdir, 'dist')
         for manifest, entries in manager.interface_manifests.items():
-            path = mozpath.join(self.environment.topobjdir, manifest)
-            with self._write_file(path) as fh:
-                for xpt in sorted(entries):
-                    fh.write('interfaces %s\n' % xpt)
+            interfaces_manifests.append(mozpath.join('$(DEPTH)', manifest))
+            for xpt in sorted(entries):
+                registered_xpt_files.add(mozpath.join(
+                    '$(DEPTH)', mozpath.dirname(manifest), xpt))
 
             if install_target.startswith('dist/'):
+                path = mozpath.join(self.environment.topobjdir, manifest)
                 path = mozpath.relpath(path, dist_dir)
                 prefix, subpath = path.split('/', 1)
                 key = 'dist_%s' % prefix
@@ -1128,9 +1156,11 @@ INSTALL_TARGETS += %(prefix)s
         obj.config = self.environment
         self._create_makefile(obj, extra=dict(
             chrome_manifests = ' '.join(chrome_manifests),
+            interfaces_manifests = ' '.join(interfaces_manifests),
             xpidl_rules=rules.getvalue(),
             xpidl_modules=' '.join(xpt_modules),
-            xpt_files=' '.join(sorted(xpt_files)),
+            xpt_files=' '.join(sorted(xpt_files - registered_xpt_files)),
+            registered_xpt_files=' '.join(sorted(registered_xpt_files)),
         ))
 
     def _process_program(self, program, backend_file):

python/mozbuild/mozbuild/backend/visualstudio.py

@@ -93,9 +93,6 @@ class VisualStudioBackend(CommonBackend):
             path=os.path.join(self._out_dir, 'mozilla.sln'))
 
     def consume_object(self, obj):
-        # Just acknowledge everything.
-        obj.ack()
-
         reldir = getattr(obj, 'relativedir', None)
 
         if hasattr(obj, 'config') and reldir not in self._paths_to_configs:
@@ -130,6 +127,9 @@ class VisualStudioBackend(CommonBackend):
             else:
                 includes.append(os.path.join('$(TopSrcDir)', reldir, p))
 
+        # Just acknowledge everything.
+        return True
+
     def _add_sources(self, reldir, obj):
         s = self._paths_to_sources.setdefault(reldir, set())
         s.update(obj.files)

python/mozbuild/mozbuild/compilation/codecomplete.py

@@ -26,12 +26,9 @@ class Introspection(MachCommandBase):
         help='Source file to display compilation flags for')
     def compileflags(self, what):
         from mozbuild.util import resolve_target_to_make
-        import shlex
+        from mozbuild.compilation import util
 
-        top_make = os.path.join(self.topobjdir, 'Makefile')
-        if not os.path.exists(top_make):
-            print('Your tree has not been built yet. Please run '
-                '|mach build| with no arguments.')
+        if not util.check_top_objdir(self.topobjdir):
             return 1
 
         path_arg = self._wrap_path_argument(what)
@@ -42,23 +39,7 @@ class Introspection(MachCommandBase):
         if make_dir is None and make_target is None:
             return 1
 
-        build_vars = {}
-
-        def on_line(line):
-            elements = [s.strip() for s in line.split('=', 1)]
-
-            if len(elements) != 2:
-                return
-
-            build_vars[elements[0]] = elements[1]
-
-        try:
-            old_logger = self.log_manager.replace_terminal_handler(None)
-            self._run_make(directory=make_dir, target='showbuild', log=False,
-                    print_directory=False, allow_parallel=False, silent=True,
-                    line_handler=on_line)
-        finally:
-            self.log_manager.replace_terminal_handler(old_logger)
+        build_vars = util.get_build_vars(make_dir, self)
 
         if what.endswith('.c'):
             name = 'COMPILE_CFLAGS'
@@ -68,20 +49,5 @@ class Introspection(MachCommandBase):
         if name not in build_vars:
             return
 
-        flags = ['-isystem', '-I', '-include', '-MF']
-        new_args = []
-        path = os.path.join(self.topobjdir, make_dir)
-        for arg in shlex.split(build_vars[name]):
-            if new_args and new_args[-1] in flags:
-                arg = os.path.normpath(os.path.join(path, arg))
-            else:
-                flag = [(f, arg[len(f):]) for f in flags + ['--sysroot=']
-                        if arg.startswith(f)]
-                if flag:
-                    flag, val = flag[0]
-                    if val:
-                        arg = flag + os.path.normpath(os.path.join(path, val))
-            new_args.append(arg)
-
-        print(' '.join(new_args))
+        print(util.get_flags(self.topobjdir, make_dir, build_vars, name))
 

python/mozbuild/mozbuild/compilation/database.py

@@ -0,0 +1,119 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+# This modules provides functionality for dealing with code completion.
+
+import os
+
+from mozbuild.base import MozbuildObject
+from mozbuild.compilation import util
+from mozbuild.backend.common import CommonBackend
+from mozbuild.frontend.data import (
+    Sources,
+    HostSources,
+    UnifiedSources,
+    GeneratedSources,
+)
+from mach.config import ConfigSettings
+from mach.logging import LoggingManager
+
+
+class CompileDBBackend(CommonBackend):
+    def _init(self):
+        CommonBackend._init(self)
+        if not util.check_top_objdir(self.environment.topobjdir):
+            raise Exception()
+
+        # The database we're going to dump out to.
+        self._db = []
+
+        # The cache for per-directory flags
+        self._flags = {}
+
+        log_manager = LoggingManager()
+        self._cmd = MozbuildObject(self.environment.topsrcdir, ConfigSettings(),
+                                   log_manager, self.environment.topobjdir)
+
+
+    def consume_object(self, obj):
+        if isinstance(obj, UnifiedSources):
+            # For unified sources, only include the unified source file.
+            # Note that unified sources are never used for host sources.
+            for f in obj.unified_source_mapping:
+                flags = self._get_dir_flags(obj.objdir)
+                self._build_db_line(obj, self.environment, f[0],
+                                    obj.canonical_suffix, flags, False)
+        elif isinstance(obj, Sources) or isinstance(obj, HostSources) or \
+             isinstance(obj, GeneratedSources):
+            # For other sources, include each source file.
+            for f in obj.files:
+                flags = self._get_dir_flags(obj.objdir)
+                self._build_db_line(obj, self.environment, f,
+                                    obj.canonical_suffix, flags,
+                                    isinstance(obj, HostSources))
+
+        return True
+
+    def consume_finished(self):
+        import json
+        # Output the database (a JSON file) to objdir/compile_commands.json
+        outputfile = os.path.join(self.environment.topobjdir, 'compile_commands.json')
+        with self._write_file(outputfile) as jsonout:
+            json.dump(self._db, jsonout, indent=0)
+
+    def _get_dir_flags(self, directory):
+        if directory in self._flags:
+            return self._flags[directory]
+
+        from mozbuild.util import resolve_target_to_make
+
+        make_dir, make_target = resolve_target_to_make(self.environment.topobjdir, directory)
+        if make_dir is None and make_target is None:
+            raise Exception('Cannot figure out the make dir and target for ' + directory)
+
+        build_vars = util.get_build_vars(directory, self._cmd)
+
+        # We only care about the following build variables.
+        for name in ('COMPILE_CFLAGS', 'COMPILE_CXXFLAGS',
+                     'COMPILE_CMFLAGS', 'COMPILE_CMMFLAGS'):
+            if name not in build_vars:
+                continue
+
+            build_vars[name] = util.get_flags(self.environment.topobjdir, directory,
+                                              build_vars, name)
+
+        self._flags[directory] = build_vars
+        return self._flags[directory]
+
+    def _build_db_line(self, obj, cenv, filename, canonical_suffix, flags, ishost):
+        # Distinguish between host and target files.
+        prefix = 'HOST_' if ishost else ''
+        if canonical_suffix == '.c':
+            compiler = cenv.substs[prefix + 'CC']
+            cflags = flags['COMPILE_CFLAGS']
+            # Add the Objective-C flags if needed.
+            if filename.endswith('.m'):
+                cflags += ' ' + flags['COMPILE_CMFLAGS']
+        elif canonical_suffix == '.cpp':
+            compiler = cenv.substs[prefix + 'CXX']
+            cflags = flags['COMPILE_CXXFLAGS']
+            # Add the Objective-C++ flags if needed.
+            if filename.endswith('.mm'):
+                cflags += ' ' + flags['COMPILE_CMMFLAGS']
+        else:
+            return
+
+        cmd = ' '.join([
+          compiler,
+          '-o', '/dev/null', '-c',
+          cflags,
+          filename,
+        ])
+
+        self._db.append({
+            'directory': obj.objdir,
+            'command': cmd,
+            'file': filename
+        })
+

python/mozbuild/mozbuild/compilation/util.py

@@ -0,0 +1,63 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this file,
+# You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import shlex
+
+def check_top_objdir(topobjdir):
+    top_make = os.path.join(topobjdir, 'Makefile')
+    if not os.path.exists(top_make):
+        print('Your tree has not been built yet. Please run '
+            '|mach build| with no arguments.')
+        return False
+    return True
+
+def get_build_vars(directory, cmd):
+    build_vars = {}
+
+    def on_line(line):
+        elements = [s.strip() for s in line.split('=', 1)]
+
+        if len(elements) != 2:
+            return
+
+        build_vars[elements[0]] = elements[1]
+
+    try:
+        old_logger = cmd.log_manager.replace_terminal_handler(None)
+        cmd._run_make(directory=directory, target='showbuild', log=False,
+                print_directory=False, allow_parallel=False, silent=True,
+                line_handler=on_line)
+    finally:
+        cmd.log_manager.replace_terminal_handler(old_logger)
+
+    return build_vars
+
+def get_flags(topobjdir, make_dir, build_vars, name):
+    flags = ['-isystem', '-I', '-include', '-MF']
+    new_args = []
+    path = os.path.join(topobjdir, make_dir)
+
+    # Take case to handle things such as the following correctly:
+    #   * -DMOZ_APP_VERSION='"40.0a1"'
+    #   * -DR_PLATFORM_INT_TYPES='<stdint.h>'
+    #   * -DAPP_ID='{ec8030f7-c20a-464f-9b0e-13a3a9e97384}
+    #   * -D__UNUSED__='__attribute__((unused))'
+    lex = shlex.shlex(build_vars[name])
+    lex.quotes = '"'
+    lex.wordchars += '+/\'"-=.*{}()[]<>'
+
+    for arg in list(lex):
+        if new_args and new_args[-1] in flags:
+            arg = os.path.normpath(os.path.join(path, arg))
+        else:
+            flag = [(f, arg[len(f):]) for f in flags + ['--sysroot=']
+                    if arg.startswith(f)]
+            if flag:
+                flag, val = flag[0]
+                if val:
+                    arg = flag + os.path.normpath(os.path.join(path, val))
+        new_args.append(arg)
+
+    return ' '.join(new_args)

python/mozbuild/mozbuild/compilation/warnings.py

@@ -12,6 +12,7 @@ import os
 import re
 
 from mozbuild.util import hash_file
+import mozpack.path as mozpath
 
 
 # Regular expression to strip ANSI color sequences from a string. This is
@@ -147,17 +148,19 @@ class WarningsDatabase(object):
             for w in value['warnings']:
                 yield w
 
-    @property
-    def type_counts(self):
+    def type_counts(self, dirpath=None):
         """Returns a mapping of warning types to their counts."""
 
         types = {}
         for value in self._files.values():
             for warning in value['warnings']:
-                count = types.get(warning['flag'], 0)
+                if dirpath and not mozpath.normsep(warning['filename']).startswith(dirpath):
+                    continue
+                flag = warning['flag']
+                count = types.get(flag, 0)
                 count += 1
 
-                types[warning['flag']] = count
+                types[flag] = count
 
         return types
 

python/mozbuild/mozbuild/config_status.py

@@ -108,7 +108,7 @@ def config_status(topobjdir='.', topsrcdir='.',
                         help='print diffs of changed files.')
     parser.add_argument('-b', '--backend', nargs='+',
                         choices=['RecursiveMake', 'AndroidEclipse', 'CppEclipse',
-                                 'VisualStudio', 'FasterMake'],
+                                 'VisualStudio', 'FasterMake', 'CompileDB'],
                         default=default_backends,
                         help='what backend to build (default: %s).' %
                         ' '.join(default_backends))
@@ -145,6 +145,9 @@ def config_status(topobjdir='.', topsrcdir='.',
         elif backend == 'FasterMake':
             from mozbuild.backend.fastermake import FasterMakeBackend
             backends_cls.append(FasterMakeBackend)
+        elif backend == 'CompileDB':
+            from mozbuild.compilation.database import CompileDBBackend
+            backends_cls.append(CompileDBBackend)
         else:
             backends_cls.append(RecursiveMakeBackend)
 

python/mozbuild/mozbuild/frontend/context.py

@@ -18,7 +18,10 @@ from __future__ import absolute_import, unicode_literals
 
 import os
 
-from collections import OrderedDict
+from collections import (
+    Counter,
+    OrderedDict,
+)
 from mozbuild.util import (
     HierarchicalStringList,
     HierarchicalStringListWithFlagsFactory,
@@ -32,9 +35,10 @@ from mozbuild.util import (
     TypedList,
     TypedNamedTuple,
 )
+
+from ..testing import all_test_flavors
 import mozpack.path as mozpath
 from types import FunctionType
-from UserString import UserString
 
 import itertools
 
@@ -43,6 +47,7 @@ class ContextDerivedValue(object):
     """Classes deriving from this one receive a special treatment in a
     Context. See Context documentation.
     """
+    __slots__ = ()
 
 
 class Context(KeyedDefaultDict):
@@ -326,8 +331,13 @@ class PathMeta(type):
             assert isinstance(context, Context)
             if isinstance(value, Path):
                 context = value.context
-        if not issubclass(cls, (SourcePath, ObjDirPath)):
-            cls = ObjDirPath if value.startswith('!') else SourcePath
+        if not issubclass(cls, (SourcePath, ObjDirPath, AbsolutePath)):
+            if value.startswith('!'):
+                cls = ObjDirPath
+            elif value.startswith('%'):
+                cls = AbsolutePath
+            else:
+                cls = SourcePath
         return super(PathMeta, cls).__call__(context, value)
 
 class Path(ContextDerivedValue, unicode):
@@ -339,6 +349,7 @@ class Path(ContextDerivedValue, unicode):
       - 'srcdir/relative/paths'
       - '!/topobjdir/relative/paths'
       - '!objdir/relative/paths'
+      - '%/filesystem/absolute/paths'
     """
     __metaclass__ = PathMeta
 
@@ -394,6 +405,8 @@ class SourcePath(Path):
     def __init__(self, context, value):
         if value.startswith('!'):
             raise ValueError('Object directory paths are not allowed')
+        if value.startswith('%'):
+            raise ValueError('Filesystem absolute paths are not allowed')
         super(SourcePath, self).__init__(context, value)
 
         if value.startswith('/'):
@@ -425,7 +438,7 @@ class ObjDirPath(Path):
     """Like Path, but limited to paths in the object directory."""
     def __init__(self, context, value=None):
         if not value.startswith('!'):
-            raise ValueError('Source paths are not allowed')
+            raise ValueError('Object directory paths must start with ! prefix')
         super(ObjDirPath, self).__init__(context, value)
 
         if value.startswith('!/'):
@@ -435,6 +448,18 @@ class ObjDirPath(Path):
         self.full_path = mozpath.normpath(path)
 
 
+class AbsolutePath(Path):
+    """Like Path, but allows arbitrary paths outside the source and object directories."""
+    def __init__(self, context, value=None):
+        if not value.startswith('%'):
+            raise ValueError('Absolute paths must start with % prefix')
+        if not os.path.isabs(value[1:]):
+            raise ValueError('Path \'%s\' is not absolute' % value[1:])
+        super(AbsolutePath, self).__init__(context, value)
+
+        self.full_path = mozpath.normpath(value[1:])
+
+
 @memoize
 def ContextDerivedTypedList(klass, base_class=List):
     """Specialized TypedList for use with ContextDerivedValue types.
@@ -464,6 +489,38 @@ def ContextDerivedTypedListWithItems(type, base_class=List):
     return _TypedListWithItems
 
 
+@memoize
+def ContextDerivedTypedRecord(*fields):
+    """Factory for objects with certain properties and dynamic
+    type checks.
+
+    This API is extremely similar to the TypedNamedTuple API,
+    except that properties may be mutated. This supports syntax like:
+
+    VARIABLE_NAME.property += [
+      'item1',
+      'item2',
+    ]
+    """
+
+    class _TypedRecord(ContextDerivedValue):
+        __slots__ = tuple([name for name, _ in fields])
+
+        def __init__(self, context):
+            for fname, ftype in self._fields.items():
+                if issubclass(ftype, ContextDerivedValue):
+                    setattr(self, fname, self._fields[fname](context))
+                else:
+                    setattr(self, fname, self._fields[fname]())
+
+        def __setattr__(self, name, value):
+            if name in self._fields and not isinstance(value, self._fields[name]):
+                value = self._fields[name](value)
+            object.__setattr__(self, name, value)
+
+    _TypedRecord._fields = dict(fields)
+    return _TypedRecord
+
 BugzillaComponent = TypedNamedTuple('BugzillaComponent',
                         [('product', unicode), ('component', unicode)])
 
@@ -471,6 +528,14 @@ WebPlatformTestManifest = TypedNamedTuple("WebPlatformTestManifest",
                                           [("manifest_path", unicode),
                                            ("test_root", unicode)])
 
+OrderedSourceList = ContextDerivedTypedList(SourcePath, StrictOrderingOnAppendList)
+OrderedTestFlavorList = TypedList(Enum(*all_test_flavors()),
+                                  StrictOrderingOnAppendList)
+OrderedStringList = TypedList(unicode, StrictOrderingOnAppendList)
+DependentTestsEntry = ContextDerivedTypedRecord(('files', OrderedSourceList),
+                                                ('tags', OrderedStringList),
+                                                ('flavors', OrderedTestFlavorList))
+
 class Files(SubContext):
     """Metadata attached to files.
 
@@ -491,8 +556,8 @@ class Files(SubContext):
     most one entity.
 
     Patterns with ``*`` or ``**`` are wildcard matches. ``*`` matches files
-    within a single directory. ``**`` matches files across several directories.
-    Here are some examples:
+    at least within a single directory. ``**`` matches files across several
+    directories.
 
     ``foo.html``
        Will match only the ``foo.html`` file in the current directory.
@@ -503,11 +568,18 @@ class Files(SubContext):
     ``foo/*.css``
        Will match all ``.css`` files in the ``foo/`` directory.
     ``bar/*``
-       Will match all files in the ``bar/`` directory but not any files in
-       child directories of ``bar/``, such as ``bar/dir1/baz``.
-    ``baz/**``
-       Will match all files in the ``baz/`` directory and all directories
-       underneath.
+       Will match all files in the ``bar/`` directory and all of its
+       children directories.
+    ``bar/**``
+       This is equivalent to ``bar/*`` above.
+    ``bar/**/foo``
+       Will match all ``foo`` files in the ``bar/`` directory and all of its
+       children directories.
+
+    The difference in behavior between ``*`` and ``**`` is only evident if
+    a pattern follows the ``*`` or ``**``. A pattern ending with ``*`` is
+    greedy. ``**`` is needed when you need an additional pattern after the
+    wildcard. e.g. ``**/foo``.
     """
 
     VARIABLES = {
@@ -532,17 +604,79 @@ class Files(SubContext):
 
             See :ref:`mozbuild_files_metadata_finalizing` for more info.
             """, None),
+        'IMPACTED_TESTS': (DependentTestsEntry, list,
+            """File patterns, tags, and flavors for tests relevant to these files.
+
+            Maps source files to the tests potentially impacted by those files.
+            Tests can be specified by file pattern, tag, or flavor.
+
+            For example:
+
+            with Files('runtests.py'):
+               IMPACTED_TESTS.files += [
+                   '**',
+               ]
+
+            in testing/mochitest/moz.build will suggest that any of the tests
+            under testing/mochitest may be impacted by a change to runtests.py.
+
+            File patterns may be made relative to the topsrcdir with a leading
+            '/', so
+
+            with Files('httpd.js'):
+               IMPACTED_TESTS.files += [
+                   '/testing/mochitest/tests/Harness_sanity/**',
+               ]
+
+            in netwerk/test/httpserver/moz.build will suggest that any change to httpd.js
+            will be relevant to the mochitest sanity tests.
+
+            Tags and flavors are sorted string lists (flavors are limited to valid
+            values).
+
+            For example:
+
+            with Files('toolkit/devtools/*'):
+                IMPACTED_TESTS.tags += [
+                    'devtools',
+                ]
+
+            in the root moz.build would suggest that any test tagged 'devtools' would
+            potentially be impacted by a change to a file under toolkit/devtools, and
+
+            with Files('dom/base/nsGlobalWindow.cpp'):
+                IMPACTED_TESTS.flavors += [
+                    'mochitest',
+                ]
+
+            Would suggest that nsGlobalWindow.cpp is potentially relevant to
+            any plain mochitest.
+            """, None),
     }
 
     def __init__(self, parent, pattern=None):
         super(Files, self).__init__(parent)
         self.pattern = pattern
         self.finalized = set()
+        self.test_files = set()
+        self.test_tags = set()
+        self.test_flavors = set()
 
     def __iadd__(self, other):
         assert isinstance(other, Files)
 
+        self.test_files |= other.test_files
+        self.test_tags |= other.test_tags
+        self.test_flavors |= other.test_flavors
+
         for k, v in other.items():
+            if k == 'IMPACTED_TESTS':
+                self.test_files |= set(mozpath.relpath(e.full_path, e.context.config.topsrcdir)
+                                       for e in v.files)
+                self.test_tags |= set(v.tags)
+                self.test_flavors |= set(v.flavors)
+                continue
+
             # Ignore updates to finalized flags.
             if k in self.finalized:
                 continue
@@ -568,6 +702,49 @@ class Files(SubContext):
 
         return d
 
+    @staticmethod
+    def aggregate(files):
+        """Given a mapping of path to Files, obtain aggregate results.
+
+        Consumers may want to extract useful information from a collection of
+        Files describing paths. e.g. given the files info data for N paths,
+        recommend a single bug component based on the most frequent one. This
+        function provides logic for deriving aggregate knowledge from a
+        collection of path File metadata.
+
+        Note: the intent of this function is to operate on the result of
+        :py:func:`mozbuild.frontend.reader.BuildReader.files_info`. The
+        :py:func:`mozbuild.frontend.context.Files` instances passed in are
+        thus the "collapsed" (``__iadd__``ed) results of all ``Files`` from all
+        moz.build files relevant to a specific path, not individual ``Files``
+        instances from a single moz.build file.
+        """
+        d = {}
+
+        bug_components = Counter()
+
+        for f in files.values():
+            bug_component = f.get('BUG_COMPONENT')
+            if bug_component:
+                bug_components[bug_component] += 1
+
+        d['bug_component_counts'] = []
+        for c, count in bug_components.most_common():
+            component = (c.product, c.component)
+            d['bug_component_counts'].append((c, count))
+
+            if 'recommended_bug_component' not in d:
+                d['recommended_bug_component'] = component
+                recommended_count = count
+            elif count == recommended_count:
+                # Don't recommend a component if it doesn't have a clear lead.
+                d['recommended_bug_component'] = None
+
+        # In case no bug components.
+        d.setdefault('recommended_bug_component', None)
+
+        return d
+
 
 # This defines functions that create sub-contexts.
 #
@@ -626,12 +803,41 @@ VARIABLES = {
         file.
         """, 'export'),
 
-    'ANDROID_RES_DIRS': (List, list,
+    'ANDROID_APK_NAME': (unicode, unicode,
+        """The name of an Android APK file to generate.
+        """, 'export'),
+
+    'ANDROID_APK_PACKAGE': (unicode, unicode,
+        """The name of the Android package to generate R.java for, like org.mozilla.gecko.
+        """, 'export'),
+
+    'ANDROID_EXTRA_PACKAGES': (StrictOrderingOnAppendList, list,
+        """The name of extra Android packages to generate R.java for, like ['org.mozilla.other'].
+        """, 'export'),
+
+    'ANDROID_EXTRA_RES_DIRS': (ContextDerivedTypedListWithItems(Path, List), list,
+        """Android extra package resource directories.
+
+        This variable contains a list of directories containing static files
+        to package into a 'res' directory and merge into an APK file.  These
+        directories are packaged into the APK but are assumed to be static
+        unchecked dependencies that should not be otherwise re-distributed.
+        """, 'export'),
+
+    'ANDROID_RES_DIRS': (ContextDerivedTypedListWithItems(Path, List), list,
         """Android resource directories.
 
-        This variable contains a list of directories, each relative to
-        the srcdir, containing static files to package into a 'res'
-        directory and merge into an APK file.
+        This variable contains a list of directories containing static
+        files to package into a 'res' directory and merge into an APK
+        file.
+        """, 'export'),
+
+    'ANDROID_ASSETS_DIRS': (ContextDerivedTypedListWithItems(Path, List), list,
+        """Android assets directories.
+
+        This variable contains a list of directories containing static
+        files to package into an 'assets' directory and merge into an
+        APK file.
         """, 'export'),
 
     'ANDROID_ECLIPSE_PROJECT_TARGETS': (dict, dict,

python/mozbuild/mozbuild/frontend/data.py

@@ -28,16 +28,14 @@ from ..util import (
     group_unified_files,
 )
 
+from ..testing import (
+    all_test_flavors,
+)
+
 
 class TreeMetadata(object):
     """Base class for all data being captured."""
 
-    def __init__(self):
-        self._ack = False
-
-    def ack(self):
-        self._ack = True
-
 
 class ContextDerived(TreeMetadata):
     """Build object derived from a single Context instance.
@@ -73,6 +71,12 @@ class ContextDerived(TreeMetadata):
 
         self.config = context.config
 
+        self._context = context
+
+    @property
+    def install_target(self):
+        return self._context['FINAL_TARGET']
+
     @property
     def relobjdir(self):
         return mozpath.relpath(self.objdir, self.topobjdir)
@@ -148,7 +152,6 @@ class XPIDLFile(ContextDerived):
     __slots__ = (
         'add_to_manifest',
         'basename',
-        'install_target',
         'source_path',
     )
 
@@ -160,8 +163,6 @@ class XPIDLFile(ContextDerived):
         self.module = module
         self.add_to_manifest = add_to_manifest
 
-        self.install_target = context['FINAL_TARGET']
-
 class BaseDefines(ContextDerived):
     """Context derived container object for DEFINES/HOST_DEFINES,
     which are OrderedDicts.
@@ -619,6 +620,8 @@ class TestManifest(ContextDerived):
             install_prefix=None, relpath=None, dupe_manifest=False):
         ContextDerived.__init__(self, context)
 
+        assert flavor in all_test_flavors()
+
         self.path = path
         self.directory = mozpath.dirname(path)
         self.manifest = manifest
@@ -959,3 +962,53 @@ class AndroidEclipseProjectData(object):
         cpe.ignore_warnings = ignore_warnings
         self._classpathentries.append(cpe)
         return cpe
+
+
+class AndroidResDirs(ContextDerived):
+    """Represents Android resource directories."""
+
+    __slots__ = (
+        'paths',
+    )
+
+    def __init__(self, context, paths):
+        ContextDerived.__init__(self, context)
+        self.paths = paths
+
+class AndroidAssetsDirs(ContextDerived):
+    """Represents Android assets directories."""
+
+    __slots__ = (
+        'paths',
+    )
+
+    def __init__(self, context, paths):
+        ContextDerived.__init__(self, context)
+        self.paths = paths
+
+class AndroidExtraResDirs(ContextDerived):
+    """Represents Android extra resource directories.
+
+    Extra resources are resources provided by libraries and including in a
+    packaged APK, but not otherwise redistributed.  In practice, this means
+    resources included in Fennec but not in GeckoView.
+    """
+
+    __slots__ = (
+        'paths',
+    )
+
+    def __init__(self, context, paths):
+        ContextDerived.__init__(self, context)
+        self.paths = paths
+
+class AndroidExtraPackages(ContextDerived):
+    """Represents Android extra packages."""
+
+    __slots__ = (
+        'packages',
+    )
+
+    def __init__(self, context, packages):
+        ContextDerived.__init__(self, context)
+        self.packages = packages

python/mozbuild/mozbuild/frontend/emitter.py

@@ -24,6 +24,10 @@ import reftest
 import mozinfo
 
 from .data import (
+    AndroidAssetsDirs,
+    AndroidExtraPackages,
+    AndroidExtraResDirs,
+    AndroidResDirs,
     BrandingFiles,
     ConfigFileSubstitution,
     ContextWrapped,
@@ -75,9 +79,14 @@ from .data import (
 
 from .reader import SandboxValidationError
 
+from ..testing import (
+    TEST_MANIFESTS,
+    REFTEST_FLAVORS,
+    WEB_PATFORM_TESTS_FLAVORS,
+)
+
 from .context import (
     Context,
-    ObjDirPath,
     SourcePath,
     ObjDirPath,
     Path,
@@ -150,8 +159,6 @@ class TreeMetadataEmitter(LoggingMixin):
             for o in objs:
                 self._object_count += 1
                 yield o
-                if not o._ack:
-                    raise Exception('Unhandled object of type %s' % type(o))
 
         for out in output:
             # Nothing in sub-contexts is currently of interest to us. Filter
@@ -542,14 +549,26 @@ class TreeMetadataEmitter(LoggingMixin):
         for obj in self._process_xpidl(context):
             yield obj
 
+        # Check for manifest declarations in EXTRA_{PP_,}COMPONENTS.
+        extras = context.get('EXTRA_COMPONENTS', []) + context.get('EXTRA_PP_COMPONENTS', [])
+        if any(e.endswith('.js') for e in extras) and \
+                not any(e.endswith('.manifest') for e in extras) and \
+                not context.get('NO_JS_MANIFEST', False):
+            raise SandboxValidationError('A .js component was specified in EXTRA_COMPONENTS '
+                                         'or EXTRA_PP_COMPONENTS without a matching '
+                                         '.manifest file.  See '
+                                         'https://developer.mozilla.org/en/XPCOM/XPCOM_changes_in_Gecko_2.0 .',
+                                         context);
+
         # Proxy some variables as-is until we have richer classes to represent
         # them. We should aim to keep this set small because it violates the
         # desired abstraction of the build definition away from makefiles.
         passthru = VariablePassthru(context)
         varlist = [
             'ALLOW_COMPILER_WARNINGS',
+            'ANDROID_APK_NAME',
+            'ANDROID_APK_PACKAGE',
             'ANDROID_GENERATED_RESFILES',
-            'ANDROID_RES_DIRS',
             'DISABLE_STL_WRAPPING',
             'EXTRA_COMPONENTS',
             'EXTRA_DSO_LDOPTS',
@@ -697,6 +716,24 @@ class TreeMetadataEmitter(LoggingMixin):
         for name, data in context.get('ANDROID_ECLIPSE_PROJECT_TARGETS', {}).items():
             yield ContextWrapped(context, data)
 
+        for (symbol, cls) in [
+                ('ANDROID_RES_DIRS', AndroidResDirs),
+                ('ANDROID_EXTRA_RES_DIRS', AndroidExtraResDirs),
+                ('ANDROID_ASSETS_DIRS', AndroidAssetsDirs)]:
+            paths = context.get(symbol)
+            if not paths:
+                continue
+            for p in paths:
+                if isinstance(p, SourcePath) and not os.path.isdir(p.full_path):
+                    raise SandboxValidationError('Directory listed in '
+                        '%s is not a directory: \'%s\'' %
+                            (symbol, p.full_path), context)
+            yield cls(context, paths)
+
+        android_extra_packages = context.get('ANDROID_EXTRA_PACKAGES')
+        if android_extra_packages:
+            yield AndroidExtraPackages(context, android_extra_packages)
+
         if passthru.variables:
             yield passthru
 
@@ -845,15 +882,15 @@ class TreeMetadataEmitter(LoggingMixin):
         for f in generated_files:
             flags = generated_files[f]
             output = f
+            inputs = []
             if flags.script:
                 method = "main"
+                script = SourcePath(context, flags.script).full_path
+
                 # Deal with cases like "C:\\path\\to\\script.py:function".
-                if not flags.script.endswith('.py') and ':' in flags.script:
-                    script, method = flags.script.rsplit(':', 1)
-                else:
-                    script = flags.script
-                script = mozpath.join(context.srcdir, script)
-                inputs = [mozpath.join(context.srcdir, i) for i in flags.inputs]
+                if '.py:' in script:
+                    script, method = script.rsplit('.py:', 1)
+                    script += '.py'
 
                 if not os.path.exists(script):
                     raise SandboxValidationError(
@@ -863,15 +900,18 @@ class TreeMetadataEmitter(LoggingMixin):
                     raise SandboxValidationError(
                         'Script for generating %s does not end in .py: %s'
                         % (f, script), context)
-                for i in inputs:
-                    if not os.path.exists(i):
+
+                for i in flags.inputs:
+                    p = Path(context, i)
+                    if (isinstance(p, SourcePath) and
+                            not os.path.exists(p.full_path)):
                         raise SandboxValidationError(
                             'Input for generating %s does not exist: %s'
-                            % (f, i), context)
+                            % (f, p.full_path), context)
+                    inputs.append(p.full_path)
             else:
                 script = None
                 method = None
-                inputs = []
             yield GeneratedFile(context, script, method, output, inputs)
 
     def _process_test_harness_files(self, context):
@@ -945,49 +985,21 @@ class TreeMetadataEmitter(LoggingMixin):
                     else 'USE_LIBS'))
 
     def _process_test_manifests(self, context):
-        # While there are multiple test manifests, the behavior is very similar
-        # across them. We enforce this by having common handling of all
-        # manifests and outputting a single class type with the differences
-        # described inside the instance.
-        #
-        # Keys are variable prefixes and values are tuples describing how these
-        # manifests should be handled:
-        #
-        #    (flavor, install_prefix, package_tests)
-        #
-        # flavor identifies the flavor of this test.
-        # install_prefix is the path prefix of where to install the files in
-        #     the tests directory.
-        # package_tests indicates whether to package test files into the test
-        #     package; suites that compile the test files should not install
-        #     them into the test package.
-        #
-        test_manifests = dict(
-            A11Y=('a11y', 'testing/mochitest', 'a11y', True),
-            BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
-            ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False),
-            JETPACK_PACKAGE=('jetpack-package', 'testing/mochitest', 'jetpack-package', True),
-            JETPACK_ADDON=('jetpack-addon', 'testing/mochitest', 'jetpack-addon', False),
-            METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
-            MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
-            MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
-            WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
-            XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
-        )
 
-        for prefix, info in test_manifests.items():
+        for prefix, info in TEST_MANIFESTS.items():
             for path in context.get('%s_MANIFESTS' % prefix, []):
                 for obj in self._process_test_manifest(context, info, path):
                     yield obj
 
-        for flavor in ('crashtest', 'reftest'):
+        for flavor in REFTEST_FLAVORS:
             for path in context.get('%s_MANIFESTS' % flavor.upper(), []):
                 for obj in self._process_reftest_manifest(context, flavor, path):
                     yield obj
 
-        for path in context.get("WEB_PLATFORM_TESTS_MANIFESTS", []):
-            for obj in self._process_web_platform_tests_manifest(context, path):
-                yield obj
+        for flavor in WEB_PATFORM_TESTS_FLAVORS:
+            for path in context.get("%s_MANIFESTS" % flavor.upper().replace('-', '_'), []):
+                for obj in self._process_web_platform_tests_manifest(context, path):
+                    yield obj
 
     def _process_test_manifest(self, context, info, manifest_path):
         flavor, install_root, install_subdir, package_tests = info
@@ -1154,11 +1166,11 @@ class TreeMetadataEmitter(LoggingMixin):
                 relpath=mozpath.join(manifest_reldir,
                     mozpath.basename(manifest_path)))
 
-        for test in sorted(manifest.files):
+        for test, source_manifest in sorted(manifest.tests):
             obj.tests.append({
                 'path': test,
                 'here': mozpath.dirname(test),
-                'manifest': manifest_full_path,
+                'manifest': source_manifest,
                 'name': mozpath.basename(test),
                 'head': '',
                 'tail': '',
@@ -1253,6 +1265,7 @@ class TreeMetadataEmitter(LoggingMixin):
 
         # Some paths have a subconfigure, yet also have a moz.build. Those
         # shouldn't end up in self._external_paths.
-        self._external_paths -= { o.relobjdir }
+        if o.objdir:
+            self._external_paths -= { o.relobjdir }
 
         yield o

python/mozbuild/mozbuild/frontend/mach_commands.py

@@ -131,6 +131,34 @@ class MozbuildFileCommands(MachCommandBase):
             print(e.message)
             return 1
 
+    @SubCommand('file-info', 'dep-tests',
+                'Show test files marked as dependencies of these source files.')
+    @CommandArgument('-r', '--rev',
+                     help='Version control revision to look up info from')
+    @CommandArgument('paths', nargs='+',
+                     help='Paths whose data to query')
+    def file_info_test_deps(self, paths, rev=None):
+        try:
+            for p, m in self._get_files_info(paths, rev=rev).items():
+                print('%s:' % mozpath.relpath(p, self.topsrcdir))
+                if m.test_files:
+                    print('\tTest file patterns:')
+                    for p in m.test_files:
+                        print('\t\t%s' % p)
+                if m.test_tags:
+                    print('\tRelevant tags:')
+                    for p in m.test_tags:
+                        print('\t\t%s' % p)
+                if m.test_flavors:
+                    print('\tRelevant flavors:')
+                    for p in m.test_flavors:
+                        print('\t\t%s' % p)
+
+        except InvalidPathException as e:
+            print(e.message)
+            return 1
+
+
     def _get_reader(self, finder):
         from mozbuild.frontend.reader import (
             BuildReader,

python/mozbuild/mozbuild/frontend/reader.py

@@ -25,7 +25,6 @@ import os
 import sys
 import textwrap
 import time
-import tokenize
 import traceback
 import types
 
@@ -41,6 +40,12 @@ from mozbuild.util import (
     ReadOnlyDefaultDict,
 )
 
+from mozbuild.testing import (
+    TEST_MANIFESTS,
+    REFTEST_FLAVORS,
+    WEB_PATFORM_TESTS_FLAVORS,
+)
+
 from mozbuild.backend.configenvironment import ConfigEnvironment
 
 from mozpack.files import FileFinder
@@ -749,7 +754,7 @@ class BuildReaderError(Exception):
             self._print_exception(inner, s)
 
     def _print_keyerror(self, inner, s):
-        if inner.args[0] not in ('global_ns', 'local_ns'):
+        if not inner.args or inner.args[0] not in ('global_ns', 'local_ns'):
             self._print_exception(inner, s)
             return
 
@@ -808,7 +813,7 @@ class BuildReaderError(Exception):
         s.write('variables and try again.\n')
 
     def _print_valueerror(self, inner, s):
-        if inner.args[0] not in ('global_ns', 'local_ns'):
+        if not inner.args or inner.args[0] not in ('global_ns', 'local_ns'):
             self._print_exception(inner, s)
             return
 
@@ -1345,6 +1350,7 @@ class BuildReader(object):
         paths, _ = self.read_relevant_mozbuilds(paths)
 
         r = {}
+        test_ctx_reader = TestContextReader(self.config)
 
         for path, ctxs in paths.items():
             flags = Files(Context())
@@ -1363,6 +1369,62 @@ class BuildReader(object):
                         ('*' in pattern and mozpath.match(relpath, pattern)):
                     flags += ctx
 
+            if not any([flags.test_tags, flags.test_files, flags.test_flavors]):
+                flags += test_ctx_reader.test_defaults_for_path(path, ctxs)
+
             r[path] = flags
 
         return r
+
+
+class TestContextReader(object):
+    """Helper to extract test patterns defaults from moz.build files.
+
+    Given paths of interest and relevant contexts, populates a Files
+    object with patterns matching all tests mentioned in the given
+    contexts.
+    """
+    def __init__(self, config):
+        self.config = config
+
+        # This names the context keys that will end up emitting a test
+        # manifest.
+        self._test_manifest_contexts = set(
+            ['%s_MANIFESTS' % key for key in TEST_MANIFESTS] +
+            ['%s_MANIFESTS' % flavor.upper() for flavor in REFTEST_FLAVORS] +
+            ['%s_MANIFESTS' % flavor.upper().replace('-', '_') for flavor in WEB_PATFORM_TESTS_FLAVORS] +
+            # The emitter requires JAR_MANIFESTS in contexts if they exist on
+            # disk, so include them here.
+            ['JAR_MANIFESTS']
+        )
+
+
+    def test_defaults_for_path(self, path, ctxs):
+        # Using the emitter here creates a circular import (and crosses some
+        # abstraction boundaries), but it's a convenient way to get the build
+        # system's view of tests.
+        # Bug 1203266 tracks features that would allow improving this situation
+        # and removing this abuse of the TreeMetadataEmitter
+        from .emitter import TreeMetadataEmitter, TestManifest
+        emitter = TreeMetadataEmitter(self.config)
+
+        result_context = Files(Context())
+
+        for ctx in ctxs:
+            test_context = Context(VARIABLES, self.config)
+            test_context.main_path = ctx.main_path
+
+            # Clone just the keys that will result in test manifests.
+            manifest_keys = [key for key in ctx if key in self._test_manifest_contexts]
+            for key in manifest_keys:
+                test_context[key] = ctx[key]
+            for obj in emitter.emit_from_context(test_context):
+                if isinstance(obj, TestManifest):
+                    for t in obj.tests:
+                        if 'relpath' not in t:
+                            # wpt manifests do not generate relpaths (bug 1207678).
+                            t['relpath'] = mozpath.relpath(t['path'],
+                                                           self.config.topsrcdir)
+                        # Pull in the entire directory of tests.
+                        result_context.test_files.add(mozpath.dirname(t['relpath']) + '/**')
+        return result_context

python/mozbuild/mozbuild/mach_commands.py

@@ -528,7 +528,7 @@ class Build(MachCommandBase):
     # conditions, but that is for another day.
     @CommandArgument('-b', '--backend', nargs='+',
         choices=['RecursiveMake', 'AndroidEclipse', 'CppEclipse',
-                 'VisualStudio', 'FasterMake'],
+                 'VisualStudio', 'FasterMake', 'CompileDB'],
         help='Which backend to build.')
     def build_backend(self, backend, diff=False):
         python = self.virtualenv_manager.python_path
@@ -645,13 +645,22 @@ class Warnings(MachCommandBase):
 
     @Command('warnings-summary', category='post-build',
         description='Show a summary of compiler warnings.')
+    @CommandArgument('-C', '--directory', default=None,
+        help='Change to a subdirectory of the build directory first.')
     @CommandArgument('report', default=None, nargs='?',
         help='Warnings report to display. If not defined, show the most '
             'recent report.')
-    def summary(self, report=None):
+    def summary(self, directory=None, report=None):
         database = self.database
 
-        type_counts = database.type_counts
+        if directory:
+            dirpath = self.join_ensure_dir(self.topsrcdir, directory)
+            if not dirpath:
+                return 1
+        else:
+            dirpath = None
+
+        type_counts = database.type_counts(dirpath)
         sorted_counts = sorted(type_counts.iteritems(),
             key=operator.itemgetter(1))
 
@@ -664,19 +673,41 @@ class Warnings(MachCommandBase):
 
     @Command('warnings-list', category='post-build',
         description='Show a list of compiler warnings.')
+    @CommandArgument('-C', '--directory', default=None,
+        help='Change to a subdirectory of the build directory first.')
+    @CommandArgument('--flags', default=None, nargs='+',
+        help='Which warnings flags to match.')
     @CommandArgument('report', default=None, nargs='?',
         help='Warnings report to display. If not defined, show the most '
             'recent report.')
-    def list(self, report=None):
+    def list(self, directory=None, flags=None, report=None):
         database = self.database
 
         by_name = sorted(database.warnings)
 
-        for warning in by_name:
-            filename = warning['filename']
+        topsrcdir = mozpath.normpath(self.topsrcdir)
 
-            if filename.startswith(self.topsrcdir):
-                filename = filename[len(self.topsrcdir) + 1:]
+        if directory:
+            directory = mozpath.normsep(directory)
+            dirpath = self.join_ensure_dir(topsrcdir, directory)
+            if not dirpath:
+                return 1
+
+        if flags:
+            # Flatten lists of flags.
+            flags = set(itertools.chain(*[flaglist.split(',') for flaglist in flags]))
+
+        for warning in by_name:
+            filename = mozpath.normsep(warning['filename'])
+
+            if filename.startswith(topsrcdir):
+                filename = filename[len(topsrcdir) + 1:]
+
+            if directory and not filename.startswith(directory):
+                continue
+
+            if flags and warning['flag'] not in flags:
+                continue
 
             if warning['column'] is not None:
                 print('%s:%d:%d [%s] %s' % (filename, warning['line'],
@@ -685,6 +716,16 @@ class Warnings(MachCommandBase):
                 print('%s:%d [%s] %s' % (filename, warning['line'],
                     warning['flag'], warning['message']))
 
+    def join_ensure_dir(self, dir1, dir2):
+        dir1 = mozpath.normpath(dir1)
+        dir2 = mozpath.normsep(dir2)
+        joined_path = mozpath.join(dir1, dir2)
+        if os.path.isdir(joined_path):
+            return joined_path
+        else:
+            print('Specified directory not found.')
+            return None
+
 @CommandProvider
 class GTestCommands(MachCommandBase):
     @Command('gtest', category='testing',

python/mozbuild/mozbuild/test/backend/data/branding-files/moz.build

@@ -0,0 +1,14 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+BRANDING_FILES += [
+    'app.ico',
+    'bar.ico',
+    'sub/quux.png',
+]
+BRANDING_FILES['app.ico'].source = 'bar.ico'
+
+BRANDING_FILES.icons += [
+    'foo.ico',
+]
+

python/mozbuild/mozbuild/test/backend/data/variable_passthru/moz.build

@@ -2,7 +2,7 @@
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
-EXTRA_COMPONENTS = ['bar.js', 'foo.js']
+EXTRA_COMPONENTS = ['bar.js', 'dummy.manifest', 'foo.js']
 EXTRA_PP_COMPONENTS = ['bar.pp.js', 'foo.pp.js']
 
 NO_VISIBILITY_FLAGS = True

python/mozbuild/mozbuild/test/backend/test_recursivemake.py

@@ -268,6 +268,7 @@ class TestRecursiveMakeBackend(BackendTester):
             ],
             'EXTRA_COMPONENTS': [
                 'EXTRA_COMPONENTS += bar.js',
+                'EXTRA_COMPONENTS += dummy.manifest',
                 'EXTRA_COMPONENTS += foo.js',
             ],
             'EXTRA_PP_COMPONENTS': [
@@ -417,6 +418,19 @@ class TestRecursiveMakeBackend(BackendTester):
         self.assertIn('res/tests/test.manifest', m)
         self.assertIn('res/tests/extra.manifest', m)
 
+    def test_branding_files(self):
+        """Ensure BRANDING_FILES is handled properly."""
+        env = self._consume('branding-files', RecursiveMakeBackend)
+
+        #BRANDING_FILES should appear in the dist_branding install manifest.
+        m = InstallManifest(path=os.path.join(env.topobjdir,
+            '_build_manifests', 'install', 'dist_branding'))
+        self.assertEqual(len(m), 4)
+        self.assertIn('app.ico', m)
+        self.assertIn('bar.ico', m)
+        self.assertIn('quux.png', m)
+        self.assertIn('icons/foo.ico', m)
+
     def test_js_preference_files(self):
         """Ensure PREF_JS_EXPORTS is written out correctly."""
         env = self._consume('js_preference_files', RecursiveMakeBackend)

python/mozbuild/mozbuild/test/frontend/data/android-res-dirs/moz.build

@@ -0,0 +1,9 @@
+# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*-
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+ANDROID_RES_DIRS += [
+    '/dir1',
+    '!/dir2',
+    '%/dir3',
+]

python/mozbuild/mozbuild/test/frontend/data/branding-files/moz.build

@@ -0,0 +1,15 @@
+# Any copyright is dedicated to the Public Domain.
+# http://creativecommons.org/publicdomain/zero/1.0/
+
+BRANDING_FILES += [
+    'app.ico',
+    'bar.ico',
+    'baz.png',
+    'foo.xpm',
+]
+BRANDING_FILES['app.ico'].source = 'test/bar.ico'
+
+BRANDING_FILES.icons += [
+    'quux.icns',
+]
+

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/moz.build

@@ -0,0 +1,5 @@
+XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell/xpcshell.ini']
+
+EXTRA_JS_MODULES += [
+    'module.js',
+]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/default/tests/xpcshell/xpcshell.ini

@@ -0,0 +1 @@
+[test_default_mod.js]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/moz.build

@@ -0,0 +1,4 @@
+DIRS += [
+  'default',
+  'simple',
+]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/browser/browser.ini

@@ -0,0 +1 @@
+[test_mod.js]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/moz.build

@@ -0,0 +1,22 @@
+with Files('src/*'):
+    IMPACTED_TESTS.files += [
+        'tests/test_general.html',
+    ]
+
+with Files('src/module.jsm'):
+    IMPACTED_TESTS.files += [
+        'browser/**.js',
+    ]
+
+with Files('base.cpp'):
+    IMPACTED_TESTS.files += [
+        '/default/tests/xpcshell/test_default_mod.js',
+        'tests/*',
+    ]
+
+
+MOCHITEST_MANIFESTS += ['tests/mochitest.ini']
+BROWSER_CHROME_MANIFESTS += ['browser/browser.ini']
+
+UNIFIED_SOURCES += ['base.cpp']
+DIRS += ['src']

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/src/moz.build

@@ -0,0 +1,3 @@
+EXTRA_JS_MODULES += [
+  'module.jsm',
+]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/mochitest.ini

@@ -0,0 +1,2 @@
+[test_general.html]
+[test_specific.html]

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/simple/tests/moz.build

@@ -0,0 +1 @@
+MOCHITEST_MANIFESTS += ['mochitest.ini']

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/moz.build

@@ -0,0 +1,15 @@
+with Files('src/submodule/**'):
+     IMPACTED_TESTS.tags += [
+         'submodule',
+     ]
+
+with Files('src/bar.jsm'):
+    IMPACTED_TESTS.flavors += [
+        'browser-chrome',
+    ]
+    IMPACTED_TESTS.files += [
+        '**.js',
+    ]
+
+MOCHITEST_MANIFESTS += ['tests/mochitest.ini']
+XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell.ini']

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/mochitest.ini

@@ -0,0 +1,3 @@
+[test_simple.html]
+[test_specific.html]
+tags = submodule

python/mozbuild/mozbuild/test/frontend/data/files-test-metadata/tagged/tests/xpcshell.ini

@@ -0,0 +1 @@
+[test_bar.js]

python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/included-reftest.list

@@ -0,0 +1 @@
+!= reftest2.html reftest2-ref.html

python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/moz.build

@@ -0,0 +1 @@
+REFTEST_MANIFESTS += ['reftest.list']

python/mozbuild/mozbuild/test/frontend/data/test-manifest-emitted-includes/reftest.list

@@ -0,0 +1,2 @@
+== reftest1.html reftest1-ref.html
+include included-reftest.list

python/mozbuild/mozbuild/test/frontend/data/variable-passthru/moz.build

@@ -2,7 +2,7 @@
 # Any copyright is dedicated to the Public Domain.
 # http://creativecommons.org/publicdomain/zero/1.0/
 
-EXTRA_COMPONENTS=['fans.js', 'tans.js']
+EXTRA_COMPONENTS = ['dummy.manifest', 'fans.js', 'tans.js']
 EXTRA_PP_COMPONENTS=['fans.pp.js', 'tans.pp.js']
 
 DIST_INSTALL = False

python/mozbuild/mozbuild/test/frontend/test_context.py

@@ -8,9 +8,12 @@ import unittest
 from mozunit import main
 
 from mozbuild.frontend.context import (
+    AbsolutePath,
     Context,
+    ContextDerivedTypedRecord,
     ContextDerivedTypedList,
     ContextDerivedTypedListWithItems,
+    Files,
     FUNCTIONS,
     ObjDirPath,
     Path,
@@ -453,6 +456,18 @@ class TestPaths(unittest.TestCase):
         path = Path(path)
         self.assertIsInstance(path, ObjDirPath)
 
+    def test_absolute_path(self):
+        config = self.config
+        ctxt = Context(config=config)
+        ctxt.push_source(mozpath.join(config.topsrcdir, 'foo', 'moz.build'))
+
+        path = AbsolutePath(ctxt, '%/qux')
+        self.assertEqual(path, '%/qux')
+        self.assertEqual(path.full_path, '/qux')
+
+        with self.assertRaises(ValueError):
+            path = AbsolutePath(ctxt, '%qux')
+
     def test_path_with_mixed_contexts(self):
         config = self.config
         ctxt1 = Context(config=config)
@@ -565,5 +580,94 @@ class TestPaths(unittest.TestCase):
             self.assertEqual(l[p_str].foo, True)
             self.assertEqual(l[p_path].foo, True)
 
+class TestTypedRecord(unittest.TestCase):
+
+    def test_fields(self):
+        T = ContextDerivedTypedRecord(('field1', unicode),
+                                      ('field2', list))
+        inst = T(None)
+        self.assertEqual(inst.field1, '')
+        self.assertEqual(inst.field2, [])
+
+        inst.field1 = 'foo'
+        inst.field2 += ['bar']
+
+        self.assertEqual(inst.field1, 'foo')
+        self.assertEqual(inst.field2, ['bar'])
+
+        with self.assertRaises(AttributeError):
+            inst.field3 = []
+
+    def test_coercion(self):
+        T = ContextDerivedTypedRecord(('field1', unicode),
+                                      ('field2', list))
+        inst = T(None)
+        inst.field1 = 3
+        inst.field2 += ('bar',)
+        self.assertEqual(inst.field1, '3')
+        self.assertEqual(inst.field2, ['bar'])
+
+        with self.assertRaises(TypeError):
+            inst.field2 = object()
+
+
+class TestFiles(unittest.TestCase):
+    def test_aggregate_empty(self):
+        c = Context({})
+
+        files = {'moz.build': Files(c, pattern='**')}
+
+        self.assertEqual(Files.aggregate(files), {
+            'bug_component_counts': [],
+            'recommended_bug_component': None,
+        })
+
+    def test_single_bug_component(self):
+        c = Context({})
+        f = Files(c, pattern='**')
+        f['BUG_COMPONENT'] = (u'Product1', u'Component1')
+
+        files = {'moz.build': f}
+        self.assertEqual(Files.aggregate(files), {
+            'bug_component_counts': [((u'Product1', u'Component1'), 1)],
+            'recommended_bug_component': (u'Product1', u'Component1'),
+        })
+
+    def test_multiple_bug_components(self):
+        c = Context({})
+        f1 = Files(c, pattern='**')
+        f1['BUG_COMPONENT'] = (u'Product1', u'Component1')
+
+        f2 = Files(c, pattern='**')
+        f2['BUG_COMPONENT'] = (u'Product2', u'Component2')
+
+        files = {'a': f1, 'b': f2, 'c': f1}
+        self.assertEqual(Files.aggregate(files), {
+            'bug_component_counts': [
+                ((u'Product1', u'Component1'), 2),
+                ((u'Product2', u'Component2'), 1),
+            ],
+            'recommended_bug_component': (u'Product1', u'Component1'),
+        })
+
+    def test_no_recommended_bug_component(self):
+        """If there is no clear count winner, we don't recommend a bug component."""
+        c = Context({})
+        f1 = Files(c, pattern='**')
+        f1['BUG_COMPONENT'] = (u'Product1', u'Component1')
+
+        f2 = Files(c, pattern='**')
+        f2['BUG_COMPONENT'] = (u'Product2', u'Component2')
+
+        files = {'a': f1, 'b': f2}
+        self.assertEqual(Files.aggregate(files), {
+            'bug_component_counts': [
+                ((u'Product1', u'Component1'), 1),
+                ((u'Product2', u'Component2'), 1),
+            ],
+            'recommended_bug_component': None,
+        })
+
+
 if __name__ == '__main__':
     main()

python/mozbuild/mozbuild/test/frontend/test_emitter.py

@@ -10,6 +10,8 @@ import unittest
 from mozunit import main
 
 from mozbuild.frontend.data import (
+    AndroidResDirs,
+    BrandingFiles,
     ConfigFileSubstitution,
     Defines,
     DistFiles,
@@ -70,11 +72,7 @@ class TestEmitterBasic(unittest.TestCase):
 
     def read_topsrcdir(self, reader, filter_common=True):
         emitter = TreeMetadataEmitter(reader.config)
-        def ack(obj):
-            obj.ack()
-            return obj
-
-        objs = list(ack(o) for o in emitter.emit(reader.read_topsrcdir()))
+        objs = list(emitter.emit(reader.read_topsrcdir()))
         self.assertGreater(len(objs), 0)
 
         filtered = []
@@ -156,7 +154,7 @@ class TestEmitterBasic(unittest.TestCase):
         wanted = {
             'ALLOW_COMPILER_WARNINGS': True,
             'DISABLE_STL_WRAPPING': True,
-            'EXTRA_COMPONENTS': ['fans.js', 'tans.js'],
+            'EXTRA_COMPONENTS': ['dummy.manifest', 'fans.js', 'tans.js'],
             'EXTRA_PP_COMPONENTS': ['fans.pp.js', 'tans.pp.js'],
             'NO_DIST_INSTALL': True,
             'VISIBILITY_FLAGS': '',
@@ -343,6 +341,23 @@ class TestEmitterBasic(unittest.TestCase):
         overwrite = resources._children['overwrite']
         self.assertEqual(overwrite._strings, ['new.res'])
 
+    def test_branding_files(self):
+        reader = self.reader('branding-files')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 1)
+        self.assertIsInstance(objs[0], BrandingFiles)
+
+        files = objs[0].files
+
+        self.assertEqual(files._strings, ['app.ico', 'bar.ico', 'baz.png', 'foo.xpm'])
+        self.assertEqual(files['app.ico'].source, 'test/bar.ico')
+
+        self.assertIn('icons', files._children)
+        icons = files._children['icons']
+
+        self.assertEqual(icons._strings, ['quux.icns'])
+
     def test_preferences_js(self):
         reader = self.reader('js_preference_files')
         objs = self.read_topsrcdir(reader)
@@ -448,6 +463,24 @@ class TestEmitterBasic(unittest.TestCase):
         paths = sorted([v[0] for v in o.installs.values()])
         self.assertEqual(paths, expected)
 
+    def test_test_manifest_includes(self):
+        """Ensure that manifest objects from the emitter list a correct manifest.
+        """
+        reader = self.reader('test-manifest-emitted-includes')
+        [obj] = self.read_topsrcdir(reader)
+
+        # Expected manifest leafs for our tests.
+        expected_manifests = {
+            'reftest1.html': 'reftest.list',
+            'reftest1-ref.html': 'reftest.list',
+            'reftest2.html': 'included-reftest.list',
+            'reftest2-ref.html': 'included-reftest.list',
+        }
+
+        for t in obj.tests:
+            self.assertTrue(t['manifest'].endswith(expected_manifests[t['name']]))
+
+
     def test_test_manifest_keys_extracted(self):
         """Ensure all metadata from test manifests is extracted."""
         reader = self.reader('test-manifest-keys-extracted')
@@ -855,5 +888,21 @@ class TestEmitterBasic(unittest.TestCase):
             reader = self.reader('dist-files-missing')
             self.read_topsrcdir(reader)
 
+    def test_android_res_dirs(self):
+        """Test that ANDROID_RES_DIRS works properly."""
+        reader = self.reader('android-res-dirs')
+        objs = self.read_topsrcdir(reader)
+
+        self.assertEqual(len(objs), 1)
+        self.assertIsInstance(objs[0], AndroidResDirs)
+
+        # Android resource directories are ordered.
+        expected = [
+            mozpath.join(reader.config.topsrcdir, 'dir1'),
+            mozpath.join(reader.config.topobjdir, 'dir2'),
+            '/dir3',
+        ]
+        self.assertEquals([p.full_path for p in objs[0].paths], expected)
+
 if __name__ == '__main__':
     main()

python/mozbuild/mozbuild/test/frontend/test_reader.py

@@ -29,6 +29,14 @@ data_path = mozpath.join(data_path, 'data')
 
 
 class TestBuildReader(unittest.TestCase):
+    def setUp(self):
+        self._old_env = dict(os.environ)
+        os.environ.pop('MOZ_OBJDIR', None)
+
+    def tearDown(self):
+        os.environ.clear()
+        os.environ.update(self._old_env)
+
     def config(self, name, **kwargs):
         path = mozpath.join(data_path, name)
 
@@ -378,6 +386,77 @@ class TestBuildReader(unittest.TestCase):
         self.assertEqual(v['bug_component/final/subcomponent/bar']['BUG_COMPONENT'],
             BugzillaComponent('Another', 'Component'))
 
+    def test_file_test_deps(self):
+        reader = self.reader('files-test-metadata')
+
+        expected = {
+            'simple/src/module.jsm': set(['simple/tests/test_general.html',
+                                          'simple/browser/**.js']),
+            'simple/base.cpp': set(['simple/tests/*',
+                                    'default/tests/xpcshell/test_default_mod.js']),
+        }
+
+        v = reader.files_info([
+            'simple/src/module.jsm',
+            'simple/base.cpp',
+        ])
+
+        for path, pattern_set in expected.items():
+            self.assertEqual(v[path].test_files,
+                             expected[path])
+
+    def test_file_test_deps_default(self):
+        reader = self.reader('files-test-metadata')
+        v = reader.files_info([
+            'default/module.js',
+        ])
+
+        expected = {
+            'default/module.js': set(['default/tests/xpcshell/**']),
+        }
+
+        for path, pattern_set in expected.items():
+            self.assertEqual(v[path].test_files,
+                             expected[path])
+
+    def test_file_test_deps_tags(self):
+        reader = self.reader('files-test-metadata')
+        v = reader.files_info([
+            'tagged/src/bar.jsm',
+            'tagged/src/submodule/foo.js',
+        ])
+
+        expected_patterns = {
+            'tagged/src/submodule/foo.js': set([]),
+            'tagged/src/bar.jsm': set(['tagged/**.js']),
+        }
+
+        for path, pattern_set in expected_patterns.items():
+            self.assertEqual(v[path].test_files,
+                             expected_patterns[path])
+
+        expected_tags = {
+            'tagged/src/submodule/foo.js': set(['submodule']),
+            'tagged/src/bar.jsm': set([]),
+        }
+        for path, pattern_set in expected_tags.items():
+            self.assertEqual(v[path].test_tags,
+                             expected_tags[path])
+
+        expected_flavors = {
+            'tagged/src/bar.jsm': set(['browser-chrome']),
+            'tagged/src/submodule/foo.js': set([]),
+        }
+        for path, pattern_set in expected_flavors.items():
+            self.assertEqual(v[path].test_flavors,
+                             expected_flavors[path])
+
+    def test_invalid_flavor(self):
+        reader = self.reader('invalid-files-flavor')
+
+        with self.assertRaises(BuildReaderError):
+            reader.files_info(['foo.js'])
+
 
 if __name__ == '__main__':
     main()

python/mozbuild/mozbuild/testing.py

@@ -212,3 +212,51 @@ class TestResolver(MozbuildObject):
                     honor_install_to_subdir=True)
             else:
                 yield test
+
+# These definitions provide a single source of truth for modules attempting
+# to get a view of all tests for a build. Used by the emitter to figure out
+# how to read/install manifests and by test dependency annotations in Files()
+# entries to enumerate test flavors.
+
+# While there are multiple test manifests, the behavior is very similar
+# across them. We enforce this by having common handling of all
+# manifests and outputting a single class type with the differences
+# described inside the instance.
+#
+# Keys are variable prefixes and values are tuples describing how these
+# manifests should be handled:
+#
+#    (flavor, install_prefix, package_tests)
+#
+# flavor identifies the flavor of this test.
+# install_prefix is the path prefix of where to install the files in
+#     the tests directory.
+# package_tests indicates whether to package test files into the test
+#     package; suites that compile the test files should not install
+#     them into the test package.
+#
+TEST_MANIFESTS = dict(
+    A11Y=('a11y', 'testing/mochitest', 'a11y', True),
+    BROWSER_CHROME=('browser-chrome', 'testing/mochitest', 'browser', True),
+    ANDROID_INSTRUMENTATION=('instrumentation', 'instrumentation', '.', False),
+    JETPACK_PACKAGE=('jetpack-package', 'testing/mochitest', 'jetpack-package', True),
+    JETPACK_ADDON=('jetpack-addon', 'testing/mochitest', 'jetpack-addon', False),
+    METRO_CHROME=('metro-chrome', 'testing/mochitest', 'metro', True),
+    MOCHITEST=('mochitest', 'testing/mochitest', 'tests', True),
+    MOCHITEST_CHROME=('chrome', 'testing/mochitest', 'chrome', True),
+    MOCHITEST_WEBAPPRT_CONTENT=('webapprt-content', 'testing/mochitest', 'webapprtContent', True),
+    MOCHITEST_WEBAPPRT_CHROME=('webapprt-chrome', 'testing/mochitest', 'webapprtChrome', True),
+    WEBRTC_SIGNALLING_TEST=('steeplechase', 'steeplechase', '.', True),
+    XPCSHELL_TESTS=('xpcshell', 'xpcshell', '.', True),
+)
+
+# Reftests have their own manifest format and are processed separately.
+REFTEST_FLAVORS = ('crashtest', 'reftest')
+
+# Web platform tests have their own manifest format and are processed separately.
+WEB_PATFORM_TESTS_FLAVORS = ('web-platform-tests',)
+
+def all_test_flavors():
+    return ([v[0] for v in TEST_MANIFESTS.values()] +
+            list(REFTEST_FLAVORS) +
+            list(WEB_PATFORM_TESTS_FLAVORS))

python/mozbuild/mozbuild/util.py

@@ -985,3 +985,20 @@ def group_unified_files(files, unified_prefix, unified_suffix,
                                               files)):
         just_the_filenames = list(filter_out_dummy(unified_group))
         yield '%s%d.%s' % (unified_prefix, i, unified_suffix), just_the_filenames
+
+
+class DefinesAction(argparse.Action):
+    '''An ArgumentParser action to handle -Dvar[=value] type of arguments.'''
+    def __call__(self, parser, namespace, values, option_string):
+        defines = getattr(namespace, self.dest)
+        if defines is None:
+            defines = {}
+        values = values.split('=', 1)
+        if len(values) == 1:
+            name, value = values[0], 1
+        else:
+            name, value = values
+            if value.isdigit():
+                value = int(value)
+        defines[name] = value
+        setattr(namespace, self.dest, defines)

python/mozbuild/mozpack/chrome/flags.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import re
 from distutils.version import LooseVersion
 from mozpack.errors import errors

python/mozbuild/mozpack/chrome/manifest.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import re
 import os
 from urlparse import urlparse

python/mozbuild/mozpack/copier.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import os
 import stat
 

python/mozbuild/mozpack/errors.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import sys
 from contextlib import contextmanager
 

python/mozbuild/mozpack/executables.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import os
 import struct
 import subprocess

python/mozbuild/mozpack/files.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import errno
 import os
 import platform
@@ -409,12 +411,15 @@ class PreprocessedFile(BaseFile):
     File class for a file that is preprocessed. PreprocessedFile.copy() runs
     the preprocessor on the file to create the output.
     '''
-    def __init__(self, path, depfile_path, marker, defines, extra_depends=None):
+    def __init__(self, path, depfile_path, marker, defines, extra_depends=None,
+                 silence_missing_directive_warnings=False):
         self.path = path
         self.depfile = depfile_path
         self.marker = marker
         self.defines = defines
         self.extra_depends = list(extra_depends or [])
+        self.silence_missing_directive_warnings = \
+            silence_missing_directive_warnings
 
     def copy(self, dest, skip_if_older=True):
         '''
@@ -463,6 +468,7 @@ class PreprocessedFile(BaseFile):
         if self.depfile:
             deps_out = FileAvoidWrite(self.depfile)
         pp = Preprocessor(defines=self.defines, marker=self.marker)
+        pp.setSilenceDirectiveWarnings(self.silence_missing_directive_warnings)
 
         with open(self.path, 'rU') as input:
             pp.processFile(input=input, output=dest, depfile=deps_out)

python/mozbuild/mozpack/hg.py

@@ -27,6 +27,8 @@
 # do not wish to do so, delete this exception statement from your
 # version.
 
+from __future__ import absolute_import
+
 import mercurial.error as error
 import mercurial.hg as hg
 import mercurial.ui as hgui

python/mozbuild/mozpack/manifests.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from __future__ import unicode_literals
+from __future__ import absolute_import, unicode_literals
 
 from contextlib import contextmanager
 import json
@@ -156,9 +156,11 @@ class InstallManifest(object):
                 continue
 
             if record_type == self.PREPROCESS:
-                dest, source, deps, marker, defines = fields[1:]
+                dest, source, deps, marker, defines, warnings = fields[1:]
+
                 self.add_preprocess(source, dest, deps, marker,
-                    self._decode_field_entry(defines))
+                    self._decode_field_entry(defines),
+                    silence_missing_directive_warnings=bool(int(warnings)))
                 continue
 
             raise UnreadableInstallManifest('Unknown record type: %d' %
@@ -281,14 +283,21 @@ class InstallManifest(object):
         self._add_entry(mozpath.join(base, pattern, dest),
             (self.PATTERN_COPY, base, pattern, dest))
 
-    def add_preprocess(self, source, dest, deps, marker='#', defines={}):
+    def add_preprocess(self, source, dest, deps, marker='#', defines={},
+                       silence_missing_directive_warnings=False):
         """Add a preprocessed file to this manifest.
 
         ``source`` will be passed through preprocessor.py, and the output will be
         written to ``dest``.
         """
-        self._add_entry(dest,
-            (self.PREPROCESS, source, deps, marker, self._encode_field_entry(defines)))
+        self._add_entry(dest, (
+            self.PREPROCESS,
+            source,
+            deps,
+            marker,
+            self._encode_field_entry(defines),
+            '1' if silence_missing_directive_warnings else '0',
+        ))
 
     def _add_entry(self, dest, entry):
         if dest in self._dests:
@@ -296,12 +305,15 @@ class InstallManifest(object):
 
         self._dests[dest] = entry
 
-    def populate_registry(self, registry):
+    def populate_registry(self, registry, defines_override={}):
         """Populate a mozpack.copier.FileRegistry instance with data from us.
 
         The caller supplied a FileRegistry instance (or at least something that
         conforms to its interface) and that instance is populated with data
         from this manifest.
+
+        Defines can be given to override the ones in the manifest for
+        preprocessing.
         """
         for dest in sorted(self._dests):
             entry = self._dests[dest]
@@ -340,11 +352,15 @@ class InstallManifest(object):
                 continue
 
             if install_type == self.PREPROCESS:
+                defines = self._decode_field_entry(entry[4])
+                if defines_override:
+                    defines.update(defines_override)
                 registry.add(dest, PreprocessedFile(entry[1],
                     depfile_path=entry[2],
                     marker=entry[3],
-                    defines=self._decode_field_entry(entry[4]),
-                    extra_depends=self._source_files))
+                    defines=defines,
+                    extra_depends=self._source_files,
+                    silence_missing_directive_warnings=bool(int(entry[5]))))
 
                 continue
 

python/mozbuild/mozpack/mozjar.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 from io import BytesIO
 import struct
 import zlib

python/mozbuild/mozpack/packager/__init__.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 from mozbuild.preprocessor import Preprocessor
 import re
 import os

python/mozbuild/mozpack/packager/formats.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 from mozpack.chrome.manifest import (
     Manifest,
     ManifestInterfaces,

python/mozbuild/mozpack/packager/l10n.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 '''
 Replace localized parts of a packaged directory with data from a langpack
 directory.

python/mozbuild/mozpack/packager/unpack.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import mozpack.path as mozpath
 from mozpack.files import (
     FileFinder,

python/mozbuild/mozpack/path.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 import posixpath
 import os
 import re

python/mozbuild/mozpack/unify.py

@@ -2,6 +2,8 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+from __future__ import absolute_import
+
 from mozpack.files import (
     BaseFinder,
     JarFinder,