import changes from `dev' branch of rmottola/Arctic-Fox:

- bug 890026 - Add mozcrash.kill_and_get_minidump r=jimm (11fe69e302)
- Bug 1162115 - Bump mozdevice to 0.45, r=wlach (4b5891b54e)
- Bug 1140145 - Update to latest wptrunner, a=testonly (a7860252bd)
- Bug 1146321 - Update to latest wptrunner, a=testonly (c81ea8eddd)
-  Bug 115107 - Update to version of wptrunner that allows prefs to be set, r=Ms2ger, ahal (972f8c55bd)
- Bug 1154691 - Update wptrunner to remove old exception type, a=testonly (f96ce919b1)
- Bug 1150821 - Update to latest wptrunner, a=testonly (bea754f26c)
- Bug 1153521 - Update to latest wptrunner, a=testonly (830e395995)
-  Bug 1155079 -Update to latest wptrunner, a=testonly (27cfd9c8a5)
- Bug 1163709 - Update to latest wptrunner, a=testonly (76672ace86)
- Bug 1160085 - Update to latest wptrunner, a=testonly (168e165a26)
- Bug 1157218 - Update to latest wptrunner, a=testonly (d9eabd0213)
- Bug 1171755 - Update to latest wptrunner, a=testonly (4f5b411410)
-  Bug 1139407 - [mozversion] Remove non-text formatters from command line log options. r=dhunt (32f7596553)
- Bug 1160087 - [moznetwork] Add command line interface. r=wlach (2efccde362)
- Bug 1175101 - [moznetwork] Bump version number to 0.25. r=wlachance (c8a0fa9ff2)
- Bug 1176677 - [moznetwork] ImportError: "cannot import name structured", and release version 0.26. r=davehunt (009449ec79)
- Bug 1160090 - [moznetwork] Add structured logging. r=wlach (05a7bc26bc)
- Bug 1160094 - [moznetwork] Attempt to pick most suitable IP when multiple are associated with the hostname. r=wlach (db464651e1)
- Bug 1163992 - [moznetwork] When multiple IPs are found on Windows pick the first one. r=wlachance (88207df55a)
- Bug 1146292 - [mozlog] Bump version to 2.11. r=jgraham (3f9e252ac4)
- Bug 1171032 - Log raw messages at debug level by default, r=chmanchester (1ac8fa11ff)
-  Bg 1171849 Let consumers override mozlog default formatter options, r=chmanchester (beb37921ca)
- Bug 1066643 - [mozlog] Allow users of mozlog's command line options to exclude inappropriate log types. r=jgraham (8f4758a6c0)
- Bug 1132409 - [mozlog] Create directories for log specified on the command line if not present. r=jgraham (2bda47bd25)
- Bug 1177630 - Add formatter to mozlog for producing a machine readable error summary, r=chmanchester (a5930babb0)
- Bug 1173380 - [mozprofile] cloned profiles are not cleaned (__del__ method is not called); r=ahal (9d7d931dbf)
- Bug 1173682 - [mozbase] tests do not remove created directories; r=ahal (a4b3c112ab)
- Bug 1161198 - Update mozdevice test for getLogcat; r=bc (10be1b1a85)
- Bug 1014760 - Move mozlog.structured to mozlog; Move mozlog to mozlog.unstructured, r=jgraham (8c1eba0f64)
- Bug 1176408 - Bump marionette-transport to 0.5 and marionette-driver to 0.9, r=dburns (fae313803a)
- Bug 1178778 - Bump marionette-driver to 0.10. r=automatedtester DONTBUILD (4196568a38)
- Bug 1183157 - make marionette --version flag also show the transport and driver package versions. r=dburns (e068536c58)
- Bug 1189027: Bump marionette driver to 0.11; r=ato (b1d103c9ef)
- Bug 1188826: Bump marionette client version for release; r=ato (018809aae3)
- Bug 1176882 - Don't pin marionette-transport. r=davehunt (c660f3ab7b)
- Bug 1169381 - Bump dependency for mozinfo in marionette-client to >=0.8. r=jgriffin (a6c0edc9bf)
- Bug 1190817 - marionette install from pypi is broken. r=automatedtester (c43fbcfaee)
- Bug 1163801 - Upgrade marionette-client from optparse to argparse, r=ahal (5843864209)
- Bug 1163801 - Refactor marionette's options mixin system for argparse compatibility, r=AutomatedTester (c8153ebde4)
- Bug 1197835 - Version bump marionette-client == 0.19, marionette-transport == 0.7, marionette-driver == 0.13, r=ato (4d7a79ae9a)
- Bug 1200973 - Remove unneeded app cache code from Marionette; r=jgriffin (0c2792e2a5)

build/mobile/b2gautomation.py

@@ -17,7 +17,7 @@ import traceback
 import zipfile
 
 from automation import Automation
-from mozlog.structured import get_default_logger
+from mozlog import get_default_logger
 from mozprocess import ProcessHandlerMixin
 
 

build/mobile/remoteautomation.py

@@ -13,7 +13,7 @@ import sys
 
 from automation import Automation
 from devicemanager import DMError, DeviceManager
-from mozlog.structured import get_default_logger
+from mozlog import get_default_logger
 import mozcrash
 
 # signatures for logcat messages that we don't care about much

layout/tools/reftest/b2g_desktop.py

@@ -21,7 +21,7 @@ from mozrunner import FirefoxRunner
 import mozinfo
 import mozlog
 
-log = mozlog.getLogger('REFTEST')
+log = mozlog.unstructured.getLogger('REFTEST')
 
 class B2GDesktopReftest(RefTest):
     build_type = "desktop"

testing/gtest/rungtests.py

@@ -15,7 +15,7 @@ import mozinfo
 import mozlog
 import mozprocess
 
-log = mozlog.getLogger('gtest')
+log = mozlog.unstructured.getLogger('gtest')
 
 class GTests(object):
     # Time (seconds) to wait for test process to complete

testing/luciddream/luciddream/runluciddream.py

@@ -14,7 +14,7 @@ from luciddream import LucidDreamTestCase
 from marionette import Marionette
 from marionette.runner import BaseMarionetteTestRunner
 import marionette
-from mozlog import structured
+import mozlog
 
 
 class CommandLineError(Exception):
@@ -60,7 +60,7 @@ def parse_args(in_args):
                         help='max time to wait for Marionette to be available after launching binary')
     parser.add_argument('manifest', metavar='MANIFEST', action='store',
                         help='path to manifest of tests to run')
-    structured.commandline.add_logging_group(parser)
+    mozlog.commandline.add_logging_group(parser)
 
     args = parser.parse_args(in_args)
     try:
@@ -117,7 +117,7 @@ def run(browser_path=None, b2g_desktop_path=None, emulator_path=None,
 
     kwargs["browser"] = browser
     if not "logger" in kwargs:
-        logger = structured.commandline.setup_logging(
+        logger = mozlog.commandline.setup_logging(
             "luciddream", kwargs, {"tbpl": sys.stdout})
         kwargs["logger"] = logger
 

testing/luciddream/requirements.txt

@@ -1,2 +1,2 @@
 marionette-client>=0.8.5
-mozlog
+mozlog>=3.0

testing/mach_commands.py

@@ -290,7 +290,7 @@ class MachCommands(MachCommandBase):
 
     def run_cppunit_test(self, **params):
         import mozinfo
-        from mozlog.structured import commandline
+        from mozlog import commandline
         import runcppunittests as cppunittests
 
         log = commandline.setup_logging("cppunittest",

testing/marionette/client/marionette/__init__.py

@@ -3,19 +3,18 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 
-__version__ = '0.16'
-
+__version__ = '0.19'
 
 from .marionette_test import MarionetteTestCase, MarionetteJSTestCase, CommonTestCase, expectedFailure, skip, SkipTest
 from .runner import (
         B2GTestCaseMixin,
         B2GTestResultMixin,
-        BaseMarionetteOptions,
+        BaseMarionetteArguments,
         BaseMarionetteTestRunner,
         BrowserMobProxyTestCaseMixin,
-        EnduranceOptionsMixin,
+        EnduranceArguments,
         EnduranceTestCaseMixin,
-        HTMLReportingOptionsMixin,
+        HTMLReportingArguments,
         HTMLReportingTestResultMixin,
         HTMLReportingTestRunnerMixin,
         Marionette,
@@ -23,7 +22,6 @@ from .runner import (
         MarionetteTestResult,
         MarionetteTextTestRunner,
         MemoryEnduranceTestCaseMixin,
-        OptionParser,
         TestManifest,
         TestResult,
         TestResultCollection

testing/marionette/client/marionette/marionette_test.py

@@ -24,9 +24,9 @@ from marionette_driver.errors import (
         MoveTargetOutOfBoundsException, FrameSendNotInitializedError, FrameSendFailureError
         )
 from marionette_driver.marionette import Marionette
-from mozlog.structured.structuredlog import get_default_logger
 from marionette_driver.wait import Wait
 from marionette_driver.expected import element_present, element_not_present
+from mozlog import get_default_logger
 
 
 class SkipTest(Exception):
@@ -376,7 +376,7 @@ class CommonTestCase(unittest.TestCase):
     def id(self):
         # TBPL starring requires that the "test name" field of a failure message
         # not differ over time. The test name to be used is passed to
-        # mozlog.structured via the test id, so this is overriden to maintain
+        # mozlog via the test id, so this is overriden to maintain
         # consistency.
         return self.test_name
 

testing/marionette/client/marionette/runner/__init__.py

@@ -2,15 +2,29 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from base import (
-        B2GTestResultMixin, BaseMarionetteOptions, BaseMarionetteTestRunner,
-        Marionette, MarionetteTest, MarionetteTestResult, MarionetteTextTestRunner,
-        OptionParser, TestManifest, TestResult, TestResultCollection
-        )
-from mixins import (
-        B2GTestCaseMixin, B2GTestResultMixin, EnduranceOptionsMixin,
-        EnduranceTestCaseMixin, HTMLReportingOptionsMixin, HTMLReportingTestResultMixin,
-        HTMLReportingTestRunnerMixin, MemoryEnduranceTestCaseMixin,
-        BrowserMobProxyTestCaseMixin, BrowserMobProxyOptionsMixin,
-        BrowserMobTestCase,
-        )
+from .base import (
+    B2GTestResultMixin,
+    BaseMarionetteArguments,
+    BaseMarionetteTestRunner,
+    Marionette,
+    MarionetteTest,
+    MarionetteTestResult,
+    MarionetteTextTestRunner,
+    TestManifest,
+    TestResult,
+    TestResultCollection,
+)
+
+from .mixins import (
+    B2GTestCaseMixin,
+    B2GTestResultMixin,
+    EnduranceArguments,
+    EnduranceTestCaseMixin,
+    HTMLReportingArguments,
+    HTMLReportingTestResultMixin,
+    HTMLReportingTestRunnerMixin,
+    MemoryEnduranceTestCaseMixin,
+    BrowserMobProxyTestCaseMixin,
+    BrowserMobProxyArguments,
+    BrowserMobTestCase,
+)

testing/marionette/client/marionette/runner/base.py

@@ -2,7 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from optparse import OptionParser
+from argparse import ArgumentParser
 
 import json
 import mozinfo
@@ -21,7 +21,7 @@ from manifestparser import TestManifest
 from manifestparser.filters import tags
 from marionette_driver.marionette import Marionette
 from mixins.b2g import B2GTestResultMixin, get_b2g_pid, get_dm
-from mozlog.structured.structuredlog import get_default_logger
+from mozlog import get_default_logger
 from moztest.adapters.unit import StructuredTestRunner, StructuredTestResult
 from moztest.results import TestResultCollection, TestResult, relevant_line
 import mozversion
@@ -244,259 +244,215 @@ class MarionetteTextTestRunner(StructuredTestRunner):
         return result
 
 
-class BaseMarionetteOptions(OptionParser):
+class BaseMarionetteArguments(ArgumentParser):
     socket_timeout_default = 360.0
 
     def __init__(self, **kwargs):
-        OptionParser.__init__(self, **kwargs)
-        self.parse_args_handlers = [] # Used by mixins
-        self.verify_usage_handlers = [] # Used by mixins
-        self.add_option('--emulator',
-                        action='store',
-                        dest='emulator',
+        ArgumentParser.__init__(self, **kwargs)
+        self.argument_containers = []
+        self.add_argument('tests',
+                          nargs='*',
+                          default=[],
+                          help='Tests to run.')
+        self.add_argument('--emulator',
                         choices=['x86', 'arm'],
                         help='if no --address is given, then the harness will launch a B2G emulator on which to run '
                              'emulator tests. if --address is given, then the harness assumes you are running an '
                              'emulator already, and will run the emulator tests using that emulator. you need to '
                              'specify which architecture to emulate for both cases')
-        self.add_option('--emulator-binary',
-                        action='store',
-                        dest='emulator_binary',
+        self.add_argument('--emulator-binary',
                         help='launch a specific emulator binary rather than launching from the B2G built emulator')
-        self.add_option('--emulator-img',
-                        action='store',
-                        dest='emulator_img',
+        self.add_argument('--emulator-img',
                         help='use a specific image file instead of a fresh one')
-        self.add_option('--emulator-res',
-                        action='store',
-                        dest='emulator_res',
-                        type='str',
+        self.add_argument('--emulator-res',
                         help='set a custom resolution for the emulator'
                              'Example: "480x800"')
-        self.add_option('--sdcard',
-                        action='store',
-                        dest='sdcard',
+        self.add_argument('--sdcard',
                         help='size of sdcard to create for the emulator')
-        self.add_option('--no-window',
+        self.add_argument('--no-window',
                         action='store_true',
-                        dest='no_window',
                         default=False,
                         help='when Marionette launches an emulator, start it with the -no-window argument')
-        self.add_option('--logcat-dir',
+        self.add_argument('--logcat-dir',
                         dest='logdir',
-                        action='store',
                         help='directory to store logcat dump files')
-        self.add_option('--logcat-stdout',
+        self.add_argument('--logcat-stdout',
                         action='store_true',
-                        dest='logcat_stdout',
                         default=False,
                         help='dump adb logcat to stdout')
-        self.add_option('--address',
-                        dest='address',
-                        action='store',
+        self.add_argument('--address',
                         help='host:port of running Gecko instance to connect to')
-        self.add_option('--device',
+        self.add_argument('--device',
                         dest='device_serial',
-                        action='store',
                         help='serial ID of a device to use for adb / fastboot')
-        self.add_option('--adb-host',
+        self.add_argument('--adb-host',
                         help='host to use for adb connection')
-        self.add_option('--adb-port',
+        self.add_argument('--adb-port',
                         help='port to use for adb connection')
-        self.add_option('--type',
-                        dest='type',
-                        action='store',
+        self.add_argument('--type',
                         help="the type of test to run, can be a combination of values defined in the manifest file; "
                              "individual values are combined with '+' or '-' characters. for example: 'browser+b2g' "
                              "means the set of tests which are compatible with both browser and b2g; 'b2g-qemu' means "
                              "the set of tests which are compatible with b2g but do not require an emulator. this "
                              "argument is only used when loading tests from manifest files")
-        self.add_option('--homedir',
-                        dest='homedir',
-                        action='store',
+        self.add_argument('--homedir',
                         help='home directory of emulator files')
-        self.add_option('--app',
-                        dest='app',
-                        action='store',
+        self.add_argument('--app',
                         help='application to use')
-        self.add_option('--app-arg',
+        self.add_argument('--app-arg',
                         dest='app_args',
                         action='append',
                         default=[],
                         help='specify a command line argument to be passed onto the application')
-        self.add_option('--binary',
-                        dest='binary',
-                        action='store',
+        self.add_argument('--binary',
                         help='gecko executable to launch before running the test')
-        self.add_option('--profile',
-                        dest='profile',
-                        action='store',
+        self.add_argument('--profile',
                         help='profile to use when launching the gecko process. if not passed, then a profile will be '
                              'constructed and used')
-        self.add_option('--addon',
-                        dest='addons',
+        self.add_argument('--addon',
                         action='append',
                         help="addon to install; repeat for multiple addons.")
-        self.add_option('--repeat',
-                        dest='repeat',
-                        action='store',
+        self.add_argument('--repeat',
                         type=int,
                         default=0,
                         help='number of times to repeat the test(s)')
-        self.add_option('-x', '--xml-output',
-                        action='store',
-                        dest='xml_output',
+        self.add_argument('-x', '--xml-output',
                         help='xml output')
-        self.add_option('--testvars',
-                        dest='testvars',
+        self.add_argument('--testvars',
                         action='append',
                         help='path to a json file with any test data required')
-        self.add_option('--tree',
-                        dest='tree',
-                        action='store',
+        self.add_argument('--tree',
                         default='b2g',
                         help='the tree that the revision parameter refers to')
-        self.add_option('--symbols-path',
-                        dest='symbols_path',
-                        action='store',
+        self.add_argument('--symbols-path',
                         help='absolute path to directory containing breakpad symbols, or the url of a zip file containing symbols')
-        self.add_option('--timeout',
-                        dest='timeout',
+        self.add_argument('--timeout',
                         type=int,
                         help='if a --timeout value is given, it will set the default page load timeout, search timeout and script timeout to the given value. If not passed in, it will use the default values of 30000ms for page load, 0ms for search timeout and 10000ms for script timeout')
-        self.add_option('--startup-timeout',
-                        dest='startup_timeout',
+        self.add_argument('--startup-timeout',
                         type=int,
                         default=60,
                         help='the max number of seconds to wait for a Marionette connection after launching a binary')
-        self.add_option('--shuffle',
+        self.add_argument('--shuffle',
                         action='store_true',
-                        dest='shuffle',
                         default=False,
                         help='run tests in a random order')
-        self.add_option('--shuffle-seed',
-                        dest='shuffle_seed',
+        self.add_argument('--shuffle-seed',
                         type=int,
                         default=random.randint(0, sys.maxint),
                         help='Use given seed to shuffle tests')
-        self.add_option('--total-chunks',
-                        dest='total_chunks',
+        self.add_argument('--total-chunks',
                         type=int,
                         help='how many chunks to split the tests up into')
-        self.add_option('--this-chunk',
-                        dest='this_chunk',
+        self.add_argument('--this-chunk',
                         type=int,
                         help='which chunk to run')
-        self.add_option('--sources',
-                        dest='sources',
-                        action='store',
+        self.add_argument('--sources',
                         help='path to sources.xml (Firefox OS only)')
-        self.add_option('--server-root',
-                        dest='server_root',
-                        action='store',
+        self.add_argument('--server-root',
                         help='url to a webserver or path to a document root from which content '
                         'resources are served (default: {}).'.format(os.path.join(
                             os.path.dirname(here), 'www')))
-        self.add_option('--gecko-log',
-                        dest='gecko_log',
-                        action='store',
+        self.add_argument('--gecko-log',
                         help="Define the path to store log file. If the path is"
                              " a directory, the real log file will be created"
                              " given the format gecko-(timestamp).log. If it is"
                              " a file, if will be used directly. '-' may be passed"
                              " to write to stdout. Default: './gecko.log'")
-        self.add_option('--logger-name',
-                        dest='logger_name',
-                        action='store',
+        self.add_argument('--logger-name',
                         default='Marionette-based Tests',
                         help='Define the name to associate with the logger used')
-        self.add_option('--jsdebugger',
-                        dest='jsdebugger',
+        self.add_argument('--jsdebugger',
                         action='store_true',
                         default=False,
                         help='Enable the jsdebugger for marionette javascript.')
-        self.add_option('--pydebugger',
-                        dest='pydebugger',
+        self.add_argument('--pydebugger',
                         help='Enable python post-mortem debugger when a test fails.'
                              ' Pass in the debugger you want to use, eg pdb or ipdb.')
-        self.add_option('--socket-timeout',
-                        dest='socket_timeout',
-                        action='store',
+        self.add_argument('--socket-timeout',
                         default=self.socket_timeout_default,
                         help='Set the global timeout for marionette socket operations.')
-        self.add_option('--e10s',
-                        dest='e10s',
+        self.add_argument('--e10s',
                         action='store_true',
                         default=False,
                         help='Enable e10s when running marionette tests.')
-        self.add_option('--tag',
+        self.add_argument('--tag',
                         action='append', dest='test_tags',
                         default=None,
                         help="Filter out tests that don't have the given tag. Can be "
                              "used multiple times in which case the test must contain "
                              "at least one of the given tags.")
 
+    def register_argument_container(self, container):
+        group = self.add_argument_group(container.name)
+
+        for cli, kwargs in container.args:
+            group.add_argument(*cli, **kwargs)
+
+        self.argument_containers.append(container)
+
     def parse_args(self, args=None, values=None):
-        options, tests = OptionParser.parse_args(self, args, values)
-        for handler in self.parse_args_handlers:
-            handler(options, tests, args, values)
+        args = ArgumentParser.parse_args(self, args, values)
+        for container in self.argument_containers:
+            if hasattr(container, 'parse_args_handler'):
+                container.parse_args_handler(args)
+        return args
 
-        return (options, tests)
-
-    def verify_usage(self, options, tests):
-        if not tests:
+    def verify_usage(self, args):
+        if not args.tests:
             print 'must specify one or more test files, manifests, or directories'
             sys.exit(1)
 
-        if not options.emulator and not options.address and not options.binary:
+        if not args.emulator and not args.address and not args.binary:
             print 'must specify --binary, --emulator or --address'
             sys.exit(1)
 
-        if options.emulator and options.binary:
+        if args.emulator and args.binary:
             print 'can\'t specify both --emulator and --binary'
             sys.exit(1)
 
         # default to storing logcat output for emulator runs
-        if options.emulator and not options.logdir:
-            options.logdir = 'logcat'
+        if args.emulator and not args.logdir:
+            args.logdir = 'logcat'
 
         # check for valid resolution string, strip whitespaces
         try:
-            if options.emulator_res:
-                dims = options.emulator_res.split('x')
+            if args.emulator_res:
+                dims = args.emulator_res.split('x')
                 assert len(dims) == 2
                 width = str(int(dims[0]))
                 height = str(int(dims[1]))
-                options.emulator_res = 'x'.join([width, height])
+                args.emulator_res = 'x'.join([width, height])
         except:
             raise ValueError('Invalid emulator resolution format. '
                              'Should be like "480x800".')
 
-        if options.total_chunks is not None and options.this_chunk is None:
+        if args.total_chunks is not None and args.this_chunk is None:
             self.error('You must specify which chunk to run.')
 
-        if options.this_chunk is not None and options.total_chunks is None:
+        if args.this_chunk is not None and args.total_chunks is None:
             self.error('You must specify how many chunks to split the tests into.')
 
-        if options.total_chunks is not None:
-            if not 1 <= options.total_chunks:
+        if args.total_chunks is not None:
+            if not 1 <= args.total_chunks:
                 self.error('Total chunks must be greater than 1.')
-            if not 1 <= options.this_chunk <= options.total_chunks:
-                self.error('Chunk to run must be between 1 and %s.' % options.total_chunks)
+            if not 1 <= args.this_chunk <= args.total_chunks:
+                self.error('Chunk to run must be between 1 and %s.' % args.total_chunks)
 
-        if options.jsdebugger:
-            options.app_args.append('-jsdebugger')
-            options.socket_timeout = None
+        if args.jsdebugger:
+            args.app_args.append('-jsdebugger')
+            args.socket_timeout = None
 
-        if options.e10s:
-            options.prefs = {
+        if args.e10s:
+            args.prefs = {
                 'browser.tabs.remote.autostart': True
             }
 
-        for handler in self.verify_usage_handlers:
-            handler(options, tests)
+        for container in self.argument_containers:
+            if hasattr(container, 'verify_usage_handler'):
+                container.verify_usage_handler(args)
 
-        return (options, tests)
+        return args
 
 
 class BaseMarionetteTestRunner(object):
@@ -514,7 +470,7 @@ class BaseMarionetteTestRunner(object):
                  sdcard=None, this_chunk=1, total_chunks=1, sources=None,
                  server_root=None, gecko_log=None, result_callbacks=None,
                  adb_host=None, adb_port=None, prefs=None, test_tags=None,
-                 socket_timeout=BaseMarionetteOptions.socket_timeout_default,
+                 socket_timeout=BaseMarionetteArguments.socket_timeout_default,
                  startup_timeout=None, addons=None, **kwargs):
         self.address = address
         self.emulator = emulator

testing/marionette/client/marionette/runner/mixins/__init__.py

@@ -2,17 +2,21 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-from endurance import (
-        EnduranceOptionsMixin, EnduranceTestCaseMixin, MemoryEnduranceTestCaseMixin
-        )
-from reporting import (
-        HTMLReportingOptionsMixin, HTMLReportingTestResultMixin,
-        HTMLReportingTestRunnerMixin
-        )
-from b2g import B2GTestCaseMixin, B2GTestResultMixin
-from browsermob import (
-    BrowserMobProxyTestCaseMixin,
-    BrowserMobProxyOptionsMixin,
-    BrowserMobTestCase,
-    )
+from .endurance import (
+    EnduranceArguments,
+    EnduranceTestCaseMixin,
+    MemoryEnduranceTestCaseMixin,
+)
 
+from .reporting import (
+    HTMLReportingArguments,
+    HTMLReportingTestResultMixin,
+    HTMLReportingTestRunnerMixin,
+)
+
+from .b2g import B2GTestCaseMixin, B2GTestResultMixin
+from .browsermob import (
+    BrowserMobProxyTestCaseMixin,
+    BrowserMobProxyArguments,
+    BrowserMobTestCase,
+)

testing/marionette/client/marionette/runner/mixins/browsermob.py

@@ -1,31 +1,29 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
 import os
 
 from browsermobproxy import Server
 from marionette import MarionetteTestCase
 
 
-class BrowserMobProxyOptionsMixin(object):
+class BrowserMobProxyArguments(object):
+    name = 'Browsermob Proxy'
+    args = [
+        [['--browsermob-script'],
+         {'help': 'path to the browsermob-proxy shell script or batch file',
+          }],
+        [['--browsermob-port'],
+         {'type': int,
+          'help': 'port to run the browsermob proxy on',
+          }],
+    ]
 
-    # verify_usage
-    def browsermob_verify_usage(self, options, tests):
-        if options.browsermob_script is not None:
-            if not os.path.exists(options.browsermob_script):
-                raise ValueError('%s not found' % options.browsermob_script)
-
-    def __init__(self, **kwargs):
-        # Inheriting object must call this __init__ to set up option handling
-        group = self.add_option_group('Browsermob Proxy')
-        group.add_option('--browsermob-script',
-                         action='store',
-                         dest='browsermob_script',
-                         type='string',
-                         help='path to the browsermob-proxy shell script or batch file')
-        group.add_option('--browsermob-port',
-                         action='store',
-                         dest='browsermob_port',
-                         type='int',
-                         help='port to run the browsermob proxy on')
-        self.verify_usage_handlers.append(self.browsermob_verify_usage)
+    def verify_usage_handler(self, args):
+        if args.browsermob_script is not None:
+            if not os.path.exists(args.browsermob_script):
+                raise ValueError('%s not found' % args.browsermob_script)
 
 
 class BrowserMobProxyTestCaseMixin(object):

testing/marionette/client/marionette/runner/mixins/endurance.py

@@ -7,41 +7,35 @@ import sys
 import time
 
 
-class EnduranceOptionsMixin(object):
+class EnduranceArguments(object):
+    name = 'endurance'
+    args = [
+        [['--iterations'],
+         {'type': int,
+          'metavar': 'int',
+          'help': 'iterations for endurance tests',
+          }],
+        [['--checkpoint'],
+         {'dest': 'checkpoint_interval',
+          'type': int,
+          'metavar': 'int',
+          'help': 'checkpoint interval for endurance tests',
+          }],
+    ]
 
-    # parse_args
-    def endurance_parse_args(self, options, tests, args=None, values=None):
-        if options.iterations is not None:
-            if options.checkpoint_interval is None or options.checkpoint_interval > options.iterations:
-                options.checkpoint_interval = options.iterations
+    def parse_args_handler(self, args):
+        if args.iterations is not None:
+            if args.checkpoint_interval is None or args.checkpoint_interval > args.iterations:
+                args.checkpoint_interval = args.iterations
 
-    # verify_usage
-    def endurance_verify_usage(self, options, tests):
-        if options.iterations is not None and options.iterations < 1:
+    def verify_usage_handler(self, args):
+        if args.iterations is not None and args.iterations < 1:
             raise ValueError('iterations must be a positive integer')
-        if options.checkpoint_interval is not None and options.checkpoint_interval < 1:
+        if args.checkpoint_interval is not None and args.checkpoint_interval < 1:
             raise ValueError('checkpoint interval must be a positive integer')
-        if options.checkpoint_interval and not options.iterations:
+        if args.checkpoint_interval and not args.iterations:
             raise ValueError('you must specify iterations when using checkpoint intervals')
 
-    def __init__(self, **kwargs):
-        # Inheriting object must call this __init__ to set up option handling
-        group = self.add_option_group('endurance')
-        group.add_option('--iterations',
-                         action='store',
-                         dest='iterations',
-                         type='int',
-                         metavar='int',
-                         help='iterations for endurance tests')
-        group.add_option('--checkpoint',
-                         action='store',
-                         dest='checkpoint_interval',
-                         type='int',
-                         metavar='int',
-                         help='checkpoint interval for endurance tests')
-        self.parse_args_handlers.append(self.endurance_parse_args)
-        self.verify_usage_handlers.append(self.endurance_verify_usage)
-
 
 class EnduranceTestCaseMixin(object):
     def __init__(self, *args, **kwargs):

testing/marionette/client/marionette/runner/mixins/reporting.py

@@ -11,7 +11,7 @@ import pkg_resources
 import sys
 import time
 
-from mozlog.structured.structuredlog import get_default_logger
+from mozlog import get_default_logger
 import mozversion
 from xmlgen import html
 from xmlgen import raw
@@ -223,15 +223,14 @@ class HTMLReportingTestRunnerMixin(object):
         return doc.unicode(indent=2)
 
 
-class HTMLReportingOptionsMixin(object):
-
-    def __init__(self, **kwargs):
-        group = self.add_option_group('htmlreporting')
-        group.add_option('--html-output',
-                         action='store',
-                         dest='html_output',
-                         help='html output',
-                         metavar='path')
+class HTMLReportingArguments(object):
+    name = 'htmlreporting'
+    args = [
+        [['--html-output'],
+         {'help': 'html output',
+          'metavar': 'path',
+          }],
+    ]
 
 
 class HTMLReportingTestResultMixin(object):

testing/marionette/client/marionette/runtests.py

@@ -5,13 +5,15 @@
 import sys
 
 from marionette import __version__
+from marionette_driver import __version__ as driver_version
+from marionette_transport import __version__ as transport_version
 from marionette.marionette_test import MarionetteTestCase, MarionetteJSTestCase
-from mozlog import structured
 from marionette.runner import (
     BaseMarionetteTestRunner,
-    BaseMarionetteOptions,
-    BrowserMobProxyOptionsMixin
-    )
+    BaseMarionetteArguments,
+    BrowserMobProxyArguments,
+)
+import mozlog
 
 
 class MarionetteTestRunner(BaseMarionetteTestRunner):
@@ -20,34 +22,41 @@ class MarionetteTestRunner(BaseMarionetteTestRunner):
         self.test_handlers = [MarionetteTestCase, MarionetteJSTestCase]
 
 
-class MarionetteOptions(BaseMarionetteOptions,
-                        BrowserMobProxyOptionsMixin):
+class MarionetteArguments(BaseMarionetteArguments):
     def __init__(self, **kwargs):
-        BaseMarionetteOptions.__init__(self, **kwargs)
-        BrowserMobProxyOptionsMixin.__init__(self, **kwargs)
+        BaseMarionetteArguments.__init__(self, **kwargs)
+        self.register_argument_container(BrowserMobProxyArguments())
 
 
-def startTestRunner(runner_class, options, tests):
-    if options.pydebugger:
-        MarionetteTestCase.pydebugger = __import__(options.pydebugger)
+def startTestRunner(runner_class, args):
+    if args.pydebugger:
+        MarionetteTestCase.pydebugger = __import__(args.pydebugger)
 
-    runner = runner_class(**vars(options))
+    args = vars(args)
+    tests = args.pop('tests')
+    runner = runner_class(**args)
     runner.run_tests(tests)
     return runner
 
-def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteOptions):
-    parser = parser_class(usage='%prog [options] test_file_or_dir <test_file_or_dir> ...',
-                          version='%prog ' + __version__)
-    structured.commandline.add_logging_group(parser)
-    options, tests = parser.parse_args()
-    parser.verify_usage(options, tests)
+def cli(runner_class=MarionetteTestRunner, parser_class=MarionetteArguments):
+    parser = parser_class(
+        usage='%prog [options] test_file_or_dir <test_file_or_dir> ...',
+        version="%prog {version} (using marionette-driver: {driver_version}"
+                ", marionette-transport: {transport_version})".format(
+                    version=__version__,
+                    driver_version=driver_version,
+                    transport_version=transport_version)
+    )
+    mozlog.commandline.add_logging_group(parser)
+    args = parser.parse_args()
+    parser.verify_usage(args)
 
-    logger = structured.commandline.setup_logging(
-        options.logger_name, options, {"tbpl": sys.stdout})
+    logger = mozlog.commandline.setup_logging(
+        args.logger_name, args, {"tbpl": sys.stdout})
 
-    options.logger = logger
+    args.logger = logger
 
-    runner = startTestRunner(runner_class, options, tests)
+    runner = startTestRunner(runner_class, args)
     if runner.failed > 0:
         sys.exit(10)
 

testing/marionette/client/marionette/tests/unit/test_appcache.py

@@ -1,31 +0,0 @@
-#Copyright 2007-2009 WebDriver committers
-#Copyright 2007-2009 Google Inc.
-#
-#Licensed under the Apache License, Version 2.0 (the "License");
-#you may not use this file except in compliance with the License.
-#You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing, software
-#distributed under the License is distributed on an "AS IS" BASIS,
-#WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#See the License for the specific language governing permissions and
-#limitations under the License.
-
-from marionette_driver.application_cache import ApplicationCache
-from marionette import MarionetteTestCase
-
-
-class AppCacheTests(MarionetteTestCase):
-
-    def testWeCanGetTheStatusOfTheAppCache(self):
-        test_url = self.marionette.absolute_url('html5Page')
-        self.marionette.navigate(test_url)
-        app_cache = self.marionette.application_cache
-
-        status = app_cache.status
-        while status == ApplicationCache.DOWNLOADING:
-            status = app_cache.status
-
-        self.assertEquals(ApplicationCache.UNCACHED, app_cache.status)

testing/marionette/client/marionette/tests/unit/unit-tests.ini

@@ -117,7 +117,6 @@ b2g = false
 [test_window_handles.py]
 b2g = false
 
-[test_appcache.py]
 [test_screenshot.py]
 [test_cookies.py]
 b2g = false

testing/marionette/client/requirements.txt

@@ -1,13 +1,12 @@
-marionette-transport >= 0.4
-marionette-driver >= 0.8
+marionette-driver >= 0.13
 browsermob-proxy >= 0.6.0
 manifestparser >= 1.1
 mozhttpd >= 0.7
-mozinfo >= 0.7
+mozinfo >= 0.8
 mozprocess >= 0.9
-mozrunner >= 6.2
+mozrunner >= 6.9
 mozdevice >= 0.44
-mozlog >= 2.7
+mozlog >= 3.0
 moznetwork >= 0.21
 mozcrash >= 0.5
 mozprofile >= 0.7

testing/marionette/driver/marionette_driver/__init__.py

@@ -1,9 +1,16 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+
+__version__ = '0.13'
+
+
 from marionette_driver import ( errors, by, decorators, expected, geckoinstance,
                                 gestures, keys, marionette, selection, wait,
-                                application_cache, date_time_value )
+                                date_time_value )
 from marionette_driver.by import By
 from marionette_driver.date_time_value import DateTimeValue
 from marionette_driver.gestures import smooth_scroll, pinch
 from marionette_driver.wait import Wait
 from marionette_driver.marionette import Actions
-

testing/marionette/driver/marionette_driver/application_cache.py

@@ -1,29 +0,0 @@
-# Copyright 2015 Mozilla Foundation
-# Copyright 2011 Software Freedom Conservancy.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-class ApplicationCache(object):
-    UNCACHED = 0
-    IDLE = 1
-    CHECKING = 2
-    DOWNLOADING = 3
-    UPDATE_READY = 4
-    OBSOLETE = 5
-
-    def __init__(self, driver):
-        self.driver = driver
-
-    @property
-    def status(self):
-        return self.driver._send_message("getAppCacheStatus", key="value")

testing/marionette/driver/marionette_driver/marionette.py

@@ -13,7 +13,6 @@ import warnings
 
 from contextlib import contextmanager
 
-from application_cache import ApplicationCache
 from decorators import do_crash_check
 from keys import Keys
 from marionette_transport import MarionetteTransport
@@ -1766,10 +1765,6 @@ class Marionette(object):
         """
         return self._send_message("getCookies", key="value" if self.protocol == 1 else None)
 
-    @property
-    def application_cache(self):
-        return ApplicationCache(self)
-
     def screenshot(self, element=None, highlights=None, format="base64",
                    full=True):
         """Takes a screenshot of a web element or the current frame.

testing/marionette/driver/requirements.txt

@@ -1,2 +1,2 @@
-marionette-transport == 0.4
-mozrunner >= 6.2
+marionette-transport == 0.7
+mozrunner >= 6.9

testing/marionette/driver/setup.py

@@ -1,13 +1,26 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import re
 from setuptools import setup, find_packages
 
-version = '0.8'
+THIS_DIR = os.path.dirname(os.path.realpath(__name__))
+
+
+def read(*parts):
+    with open(os.path.join(THIS_DIR, *parts)) as f:
+        return f.read()
+
+
+def get_version():
+    return re.findall("__version__ = '([\d\.]+)'",
+                      read('marionette_driver', '__init__.py'), re.M)[0]
 
-# dependencies
-with open('requirements.txt') as f:
-    deps = f.read().splitlines()
 
 setup(name='marionette_driver',
-      version=version,
+      version=get_version(),
       description="Marionette Driver",
       long_description='See http://marionette-driver.readthedocs.org/',
       classifiers=[],  # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
@@ -19,5 +32,5 @@ setup(name='marionette_driver',
       packages=find_packages(),
       include_package_data=True,
       zip_safe=False,
-      install_requires=deps,
+      install_requires=read('requirements.txt').splitlines(),
       )

testing/marionette/mach_commands.py

@@ -39,36 +39,37 @@ def run_marionette(tests, b2g_path=None, emulator=None, testtype=None,
 
     from marionette.runtests import (
         MarionetteTestRunner,
-        BaseMarionetteOptions,
+        BaseMarionetteArguments,
         startTestRunner
     )
 
-    parser = BaseMarionetteOptions()
+    parser = BaseMarionetteArguments()
     commandline.add_logging_group(parser)
-    options, args = parser.parse_args()
+    args = parser.parse_args()
 
     if not tests:
         tests = [os.path.join(topsrcdir,
-                    'testing/marionette/client/marionette/tests/unit-tests.ini')]
+                 'testing/marionette/client/marionette/tests/unit-tests.ini')]
+    args.tests = tests
 
     if b2g_path:
-        options.homedir = b2g_path
+        args.homedir = b2g_path
         if emulator:
-            options.emulator = emulator
+            args.emulator = emulator
     else:
-        options.binary = binary
-        path, exe = os.path.split(options.binary)
+        args.binary = binary
+        path, exe = os.path.split(args.binary)
 
     for k, v in kwargs.iteritems():
-        setattr(options, k, v)
+        setattr(args, k, v)
 
-    parser.verify_usage(options, tests)
+    parser.verify_usage(args)
 
-    options.logger = commandline.setup_logging("Marionette Unit Tests",
-                                               options,
-                                               {"mach": sys.stdout})
+    args.logger = commandline.setup_logging("Marionette Unit Tests",
+                                            args,
+                                            {"mach": sys.stdout})
 
-    runner = startTestRunner(MarionetteTestRunner, options, tests)
+    runner = startTestRunner(MarionetteTestRunner, args)
     if runner.failed > 0:
         return 1
 

testing/marionette/transport/marionette_transport/__init__.py

@@ -1 +1,8 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/
+
+__version__ = '0.7'
+
+
 from transport import MarionetteTransport

testing/marionette/transport/setup.py

@@ -1,6 +1,24 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import os
+import re
 from setuptools import setup, find_packages
 
-version = '0.4'
+
+THIS_DIR = os.path.dirname(os.path.realpath(__name__))
+
+
+def read(*parts):
+    with open(os.path.join(THIS_DIR, *parts)) as f:
+        return f.read()
+
+
+def get_version():
+    return re.findall("__version__ = '([\d\.]+)'",
+                      read('marionette_transport', '__init__.py'), re.M)[0]
+
 
 long_description = \
 """Marionette_ is a Mozilla project to enable remote automation in Gecko-based
@@ -14,11 +32,9 @@ points; rather it's designed to be used by a Marionette client implementation.
 .. _Marionette: https://developer.mozilla.org/en/Marionette
 .. _`Selenium Webdriver`: http://www.seleniumhq.org/projects/webdriver/"""
 
-# dependencies
-deps = []
 
 setup(name='marionette-transport',
-      version=version,
+      version=get_version(),
       description="Transport layer for Marionette client",
       long_description=long_description,
       classifiers=[], # Get strings from http://pypi.python.org/pypi?%3Aaction=list_classifiers
@@ -33,6 +49,5 @@ setup(name='marionette-transport',
       zip_safe=False,
       entry_points="""
       """,
-      install_requires=deps,
+      install_requires=[],
       )
-

testing/mochitest/mochitest_options.py

@@ -9,9 +9,9 @@ import os
 import tempfile
 
 from droid import DroidADB, DroidSUT
-from mozlog import structured
 from mozprofile import DEFAULT_PORTS
 import mozinfo
+import mozlog
 import moznetwork
 
 
@@ -1201,7 +1201,7 @@ class MochitestArgumentParser(ArgumentParser):
                 group.add_argument(*cli, **kwargs)
 
         self.set_defaults(**defaults)
-        structured.commandline.add_logging_group(self)
+        mozlog.commandline.add_logging_group(self)
 
     @property
     def containers(self):

testing/mochitest/runtests.py

@@ -55,8 +55,8 @@ from mochitest_options import MochitestArgumentParser, build_obj
 from mozprofile import Profile, Preferences
 from mozprofile.permissions import ServerLocations
 from urllib import quote_plus as encodeURIComponent
-from mozlog.structured.formatters import TbplFormatter
-from mozlog.structured import commandline
+from mozlog.formatters import TbplFormatter
+from mozlog import commandline
 from mozrunner.utils import test_environment
 import mozleak
 

testing/mochitest/runtestsb2g.py

@@ -19,7 +19,6 @@ from runtests import MochitestUtilsMixin
 from mochitest_options import MochitestArgumentParser
 from marionette import Marionette
 from mozprofile import Profile, Preferences
-from mozlog import structured
 import mozinfo
 import mozleak
 

testing/modules/StructuredLog.jsm

@@ -10,7 +10,7 @@ this.EXPORTED_SYMBOLS = [
 
 /**
  * TestLogger: Logger class generating messages compliant with the
- * structured logging protocol for tests exposed by the mozlog.structured
+ * structured logging protocol for tests exposed by mozlog
  *
  * @param name
  *        The name of the logger to instantiate.
@@ -30,8 +30,7 @@ this.StructuredLogger = function (name, dumpFun=dump, mutators=[]) {
 }
 
 /**
- * Log functions producing messages in the format specified by
- * mozlog.structured
+ * Log functions producing messages in the format specified by mozlog
  */
 StructuredLogger.prototype.testStart = function (test) {
   this._runningTests.add(test);

testing/mozbase/docs/_static/structured_example.py

@@ -3,7 +3,7 @@ import sys
 import traceback
 import types
 
-from mozlog.structured import commandline, get_default_logger
+from mozlog import commandline, get_default_logger
 
 class TestAssertion(Exception):
     pass

testing/mozbase/docs/loggingreporting.rst

@@ -8,4 +8,4 @@ less verbose. We created some libraries to make doing this easy.
 .. toctree::
    :maxdepth: 2
 
-   mozlog_structured
+   mozlog

testing/mozbase/docs/mozlog.rst

@@ -1,13 +1,13 @@
-:mod:`mozlog.structured` --- Structured logging for test output
+:mod:`mozlog` --- Structured logging for test output
 ===============================================================
 
-:py:mod:`mozlog.structured` is a library designed for logging the
+:py:mod:`mozlog` is a library designed for logging the
 execution and results of test harnesses. The internal data model is a
 stream of JSON-compatible objects, with one object per log entry. The
 default output format is line-based, with one JSON object serialized
 per line.
 
-:py:mod:`mozlog.structured` is *not* based on the stdlib logging
+:py:mod:`mozlog` is *not* based on the stdlib logging
 module, although it shares several concepts with it.
 
 One notable difference between this module and the standard logging
@@ -199,7 +199,7 @@ under test to crash).
 StructuredLogger Objects
 ------------------------
 
-.. automodule:: mozlog.structured.structuredlog
+.. automodule:: mozlog.structuredlog
   :members: set_default_logger, get_default_logger
 
 .. autoclass:: StructuredLogger
@@ -219,7 +219,7 @@ message. The typical example of this is a ``StreamHandler`` which takes
 a log message, invokes a formatter which converts the log to a string,
 and writes it to a file.
 
-.. automodule:: mozlog.structured.handlers
+.. automodule:: mozlog.handlers
 
 .. autoclass:: BaseHandler
   :members:
@@ -248,32 +248,32 @@ Formatter modules are written so that they can take raw input on stdin
 and write formatted output on stdout. This allows the formatters to be
 invoked as part of a command line for post-processing raw log files.
 
-.. automodule:: mozlog.structured.formatters.base
+.. automodule:: mozlog.formatters.base
 
 .. autoclass:: BaseFormatter
   :members:
 
-.. automodule:: mozlog.structured.formatters.unittest
+.. automodule:: mozlog.formatters.unittest
 
 .. autoclass:: UnittestFormatter
   :members:
 
-.. automodule:: mozlog.structured.formatters.xunit
+.. automodule:: mozlog.formatters.xunit
 
 .. autoclass:: XUnitFormatter
   :members:
 
-.. automodule:: mozlog.structured.formatters.html
+.. automodule:: mozlog.formatters.html
 
 .. autoclass:: HTMLFormatter
   :members:
 
-.. automodule:: mozlog.structured.formatters.machformatter
+.. automodule:: mozlog.formatters.machformatter
 
 .. autoclass:: MachFormatter
   :members:
 
-.. automodule:: mozlog.structured.formatters.tbplformatter
+.. automodule:: mozlog.formatters.tbplformatter
 
 .. autoclass:: TbplFormatter
   :members:
@@ -281,22 +281,22 @@ invoked as part of a command line for post-processing raw log files.
 Processing Log Files
 --------------------
 
-The ``mozlog.structured.reader`` module provides utilities for working
+The ``mozlog.reader`` module provides utilities for working
 with structured log files.
 
-.. automodule:: mozlog.structured.reader
+.. automodule:: mozlog.reader
   :members:
 
 Integration with argparse
 -------------------------
 
-The `mozlog.structured.commandline` module provides integration with
-the `argparse` module to provide uniform logging-related command line
-arguments to programs using `mozlog.structured`. Each known formatter
-gets a command line argument of the form ``--log-{name}``, which takes
-the name of a file to log to with that format, or ``-`` to indicate stdout.
+The `mozlog.commandline` module provides integration with the `argparse`
+module to provide uniform logging-related command line arguments to programs
+using `mozlog`. Each known formatter gets a command line argument of the form
+``--log-{name}``, which takes the name of a file to log to with that format,
+or ``-`` to indicate stdout.
 
-.. automodule:: mozlog.structured.commandline
+.. automodule:: mozlog.commandline
   :members:
 
 Simple Examples
@@ -304,8 +304,8 @@ Simple Examples
 
 Log to stdout::
 
-    from mozlog.structured import structuredlog
-    from mozlog.structured import handlers, formatters
+    from mozlog import structuredlog
+    from mozlog import handlers, formatters
     logger = structuredlog.StructuredLogger("my-test-suite")
     logger.add_handler(handlers.StreamHandler(sys.stdout,
                                               formatters.JSONFormatter()))
@@ -322,7 +322,7 @@ create a logger based on the value of those options, defaulting to
 JSON output on stdout if nothing else is supplied::
 
    import argparse
-   from mozlog.structured import commandline
+   from mozlog import commandline
 
    parser = argparse.ArgumentParser()
    # Here one would populate the parser with other options
@@ -333,7 +333,7 @@ JSON output on stdout if nothing else is supplied::
 
 Count the number of tests that timed out in a testsuite::
 
-   from mozlog.structured import reader
+   from mozlog import reader
 
    count = 0
 
@@ -373,7 +373,7 @@ set of structured logging arguments appropriate to all tools producing
 structured logging.
 
 The values of these command line arguments are used to create a
-:py:class:`mozlog.structured.StructuredLogger` object populated with the
+:py:class:`mozlog.StructuredLogger` object populated with the
 specified handlers and formatters in
 :py:func:`commandline.setup_logging`. The third argument to this
 function is the default arguments to use. In this case the default

testing/mozbase/mozcrash/mozcrash/mozcrash.py

@@ -5,7 +5,8 @@
 __all__ = [
     'check_for_crashes',
     'check_for_java_exception',
-    'log_crashes'
+    'kill_and_get_minidump',
+    'log_crashes',
 ]
 
 import glob
@@ -20,8 +21,8 @@ import zipfile
 from collections import namedtuple
 
 import mozfile
+import mozinfo
 import mozlog
-from mozlog.structured import structuredlog
 
 
 StackInfo = namedtuple("StackInfo",
@@ -35,9 +36,9 @@ StackInfo = namedtuple("StackInfo",
 
 
 def get_logger():
-    structured_logger = structuredlog.get_default_logger("mozcrash")
+    structured_logger = mozlog.get_default_logger("mozcrash")
     if structured_logger is None:
-        return mozlog.getLogger('mozcrash')
+        return mozlog.unstructured.getLogger('mozcrash')
     return structured_logger
 
 
@@ -339,3 +340,107 @@ def check_for_java_exception(logcat, quiet=False):
             break
 
     return found_exception
+
+if mozinfo.isWin:
+    import ctypes
+    import uuid
+
+    kernel32 = ctypes.windll.kernel32
+    OpenProcess = kernel32.OpenProcess
+    CloseHandle = kernel32.CloseHandle
+
+    def write_minidump(pid, dump_directory):
+        """
+        Write a minidump for a process.
+
+        :param pid: PID of the process to write a minidump for.
+        :param dump_directory: Directory in which to write the minidump.
+        """
+        PROCESS_QUERY_INFORMATION = 0x0400
+        PROCESS_VM_READ = 0x0010
+        GENERIC_READ = 0x80000000
+        GENERIC_WRITE = 0x40000000
+        CREATE_ALWAYS = 2
+        FILE_ATTRIBUTE_NORMAL = 0x80
+        INVALID_HANDLE_VALUE = -1
+
+        proc_handle = OpenProcess(PROCESS_QUERY_INFORMATION | PROCESS_VM_READ,
+                                  0, pid)
+        if not proc_handle:
+            return
+
+        file_name = os.path.join(dump_directory,
+                                 str(uuid.uuid4()) + ".dmp")
+        if not isinstance(file_name, unicode):
+            # Convert to unicode explicitly so our path will be valid as input
+            # to CreateFileW
+            file_name = unicode(file_name, sys.getfilesystemencoding())
+
+        file_handle = kernel32.CreateFileW(file_name,
+                                           GENERIC_READ | GENERIC_WRITE,
+                                           0,
+                                           None,
+                                           CREATE_ALWAYS,
+                                           FILE_ATTRIBUTE_NORMAL,
+                                           None)
+        if file_handle != INVALID_HANDLE_VALUE:
+            ctypes.windll.dbghelp.MiniDumpWriteDump(proc_handle,
+                                                    pid,
+                                                    file_handle,
+                                                    # Dump type - MiniDumpNormal
+                                                    0,
+                                                    # Exception parameter
+                                                    None,
+                                                    # User stream parameter
+                                                    None,
+                                                    # Callback parameter
+                                                    None)
+            CloseHandle(file_handle)
+        CloseHandle(proc_handle)
+
+    def kill_pid(pid):
+        """
+        Terminate a process with extreme prejudice.
+
+        :param pid: PID of the process to terminate.
+        """
+        PROCESS_TERMINATE = 0x0001
+        handle = OpenProcess(PROCESS_TERMINATE, 0, pid)
+        if handle:
+            kernel32.TerminateProcess(handle, 1)
+            CloseHandle(handle)
+else:
+    def kill_pid(pid):
+        """
+        Terminate a process with extreme prejudice.
+
+        :param pid: PID of the process to terminate.
+        """
+        os.kill(pid, signal.SIGKILL)
+
+def kill_and_get_minidump(pid, dump_directory=None):
+    """
+    Attempt to kill a process and leave behind a minidump describing its
+    execution state.
+
+    :param pid: The PID of the process to kill.
+    :param dump_directory: The directory where a minidump should be written on
+    Windows, where the dump will be written from outside the process.
+
+    On Windows a dump will be written using the MiniDumpWriteDump function
+    from DbgHelp.dll. On Linux and OS X the process will be sent a SIGABRT
+    signal to trigger minidump writing via a Breakpad signal handler. On other
+    platforms the process will simply be killed via SIGKILL.
+
+    If the process is hung in such a way that it cannot respond to SIGABRT
+    it may still be running after this function returns. In that case it
+    is the caller's responsibility to deal with killing it.
+    """
+    needs_killing = True
+    if mozinfo.isWin:
+        write_minidump(pid, dump_directory)
+    elif mozinfo.isLinux or mozinfo.isMac:
+        os.kill(pid, signal.SIGABRT)
+        needs_killing = False
+    if needs_killing:
+        kill_pid(pid)

testing/mozbase/mozcrash/setup.py

@@ -9,7 +9,7 @@ PACKAGE_VERSION = '0.14'
 
 # dependencies
 deps = ['mozfile >= 1.0',
-        'mozlog']
+        'mozlog >= 3.0']
 
 setup(name=PACKAGE_NAME,
       version=PACKAGE_VERSION,

testing/mozbase/mozcrash/tests/test.py

@@ -5,7 +5,9 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import os, unittest, subprocess, tempfile, shutil, urlparse, zipfile, StringIO
-import mozcrash, mozlog, mozhttpd
+import mozcrash
+import mozhttpd
+import mozlog.unstructured as mozlog
 
 # Make logs go away
 log = mozlog.getLogger("mozcrash", handler=mozlog.FileHandler(os.devnull))

testing/mozbase/mozdevice/mozdevice/adb.py

@@ -157,8 +157,8 @@ class ADBCommand(object):
     def _get_logger(self, logger_name):
         logger = None
         try:
-            from mozlog import structured
-            logger = structured.get_default_logger(logger_name)
+            import mozlog
+            logger = mozlog.get_default_logger(logger_name)
         except ImportError:
             pass
 

testing/mozbase/mozdevice/mozdevice/devicemanager.py

@@ -4,6 +4,7 @@
 
 import hashlib
 import mozlog
+import logging
 import os
 import posixpath
 import re
@@ -50,9 +51,9 @@ class DeviceManager(object):
 
     def __init__(self, logLevel=None, deviceRoot=None):
         try:
-            self._logger = mozlog.structured.structuredlog.get_default_logger(component="mozdevice")
+            self._logger = mozlog.get_default_logger(component="mozdevice")
             if not self._logger: # no global structured logger, fall back to reg logging
-                self._logger = mozlog.getLogger("mozdevice")
+                self._logger = mozlog.unstructured.getLogger("mozdevice")
                 if logLevel is not None:
                     self._logger.setLevel(logLevel)
         except AttributeError:
@@ -88,16 +89,16 @@ class DeviceManager(object):
     @property
     def debug(self):
         self._logger.warning("dm.debug is deprecated. Use logLevel.")
-        levels = {mozlog.DEBUG: 5, mozlog.INFO: 3, mozlog.WARNING: 2,
-                  mozlog.ERROR: 1, mozlog.CRITICAL: 0}
+        levels = {logging.DEBUG: 5, logging.INFO: 3, logging.WARNING: 2,
+                  logging.ERROR: 1, logging.CRITICAL: 0}
         return levels[self.logLevel]
 
     @debug.setter
     def debug_setter(self, newDebug):
         self._logger.warning("dm.debug is deprecated. Use logLevel.")
         newDebug = 5 if newDebug > 5 else newDebug # truncate >=5 to 5
-        levels = {5: mozlog.DEBUG, 3: mozlog.INFO, 2: mozlog.WARNING,
-                  1: mozlog.ERROR, 0: mozlog.CRITICAL}
+        levels = {5: logging.DEBUG, 3: logging.INFO, 2: logging.WARNING,
+                  1: logging.ERROR, 0: logging.CRITICAL}
         self.logLevel = levels[newDebug]
 
     @abstractmethod

testing/mozbase/mozdevice/mozdevice/devicemanagerADB.py

@@ -2,6 +2,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import logging
 import re
 import os
 import shutil
@@ -12,7 +13,6 @@ import traceback
 from devicemanager import DeviceManager, DMError
 from mozprocess import ProcessHandler
 import mozfile
-import mozlog
 
 
 class DeviceManagerADB(DeviceManager):
@@ -34,7 +34,7 @@ class DeviceManagerADB(DeviceManager):
 
     def __init__(self, host=None, port=5555, retryLimit=5, packageName='fennec',
                  adbPath='adb', deviceSerial=None, deviceRoot=None,
-                 logLevel=mozlog.ERROR, autoconnect=True, runAdbAsRoot=False,
+                 logLevel=logging.ERROR, autoconnect=True, runAdbAsRoot=False,
                  serverHost=None, serverPort=None, **kwargs):
         DeviceManager.__init__(self, logLevel=logLevel,
                                deviceRoot=deviceRoot)

testing/mozbase/mozdevice/mozdevice/devicemanagerSUT.py

@@ -3,7 +3,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import datetime
-import mozlog
+import logging
 import moznetwork
 import select
 import socket
@@ -34,10 +34,10 @@ class DeviceManagerSUT(DeviceManager):
     reboot_timeout = 600
     reboot_settling_time = 60
 
-    def __init__(self, host, port = 20701, retryLimit = 5,
-            deviceRoot = None, logLevel = mozlog.ERROR, **kwargs):
-        DeviceManager.__init__(self, logLevel = logLevel,
-                               deviceRoot = deviceRoot)
+    def __init__(self, host, port=20701, retryLimit=5, deviceRoot=None,
+                 logLevel=logging.ERROR, **kwargs):
+        DeviceManager.__init__(self, logLevel=logLevel,
+                               deviceRoot=deviceRoot)
         self.host = host
         self.port = port
         self.retryLimit = retryLimit

testing/mozbase/mozdevice/mozdevice/dmcli.py

@@ -7,6 +7,7 @@ Command-line client to control a device
 """
 
 import errno
+import logging
 import os
 import posixpath
 import StringIO
@@ -134,12 +135,12 @@ class DMCli(object):
         self.parser = argparse.ArgumentParser()
         self.add_options(self.parser)
         self.add_commands(self.parser)
-        mozlog.structured.commandline.add_logging_group(self.parser)
+        mozlog.commandline.add_logging_group(self.parser)
 
     def run(self, args=sys.argv[1:]):
         args = self.parser.parse_args()
 
-        mozlog.structured.commandline.setup_logging(
+        mozlog.commandline.setup_logging(
             'mozdevice', args, {'mach': sys.stdout})
 
         if args.dmtype == "sut" and not args.host and not args.hwid:
@@ -202,9 +203,9 @@ class DMCli(object):
         '''
         Returns a device with the specified parameters
         '''
-        logLevel = mozlog.ERROR
+        logLevel = logging.ERROR
         if verbose:
-            logLevel = mozlog.DEBUG
+            logLevel = logging.DEBUG
 
         if hwid:
             return mozdevice.DroidConnectByHWID(hwid, logLevel=logLevel)

testing/mozbase/mozdevice/setup.py

@@ -5,10 +5,10 @@
 from setuptools import setup
 
 PACKAGE_NAME = 'mozdevice'
-PACKAGE_VERSION = '0.44'
+PACKAGE_VERSION = '0.45'
 
 deps = ['mozfile >= 1.0',
-        'mozlog >= 2.1',
+        'mozlog >= 3.0',
         'moznetwork >= 0.24',
         'mozprocess >= 0.19',
        ]

testing/mozbase/mozdevice/sut_tests/dmunit.py

@@ -2,18 +2,17 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import logging
 import types
 import unittest
 
-import mozlog
-
 from mozdevice import devicemanager
 from mozdevice import devicemanagerSUT
 
 ip = ''
 port = 0
 heartbeat_port = 0
-log_level = mozlog.ERROR
+log_level = logging.ERROR
 
 class DeviceManagerTestCase(unittest.TestCase):
     """DeviceManager tests should subclass this.

testing/mozbase/mozdevice/sut_tests/runtests.py

@@ -3,13 +3,12 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 from optparse import OptionParser
+import logging
 import os
 import re
 import sys
 import unittest
 
-import mozlog
-
 import dmunit
 import genfiles
 
@@ -19,7 +18,7 @@ def main(ip, port, heartbeat_port, scripts, directory, isTestDevice, verbose):
     dmunit.port = port
     dmunit.heartbeat_port = heartbeat_port
     if verbose:
-        dmunit.log_level = mozlog.DEBUG
+        dmunit.log_level = logging.DEBUG
 
     suite = unittest.TestSuite()
 

testing/mozbase/mozdevice/tests/droidsut_launch.py

@@ -1,6 +1,6 @@
 from sut import MockAgent
 import mozdevice
-import mozlog
+import logging
 import unittest
 
 class LaunchTest(unittest.TestCase):
@@ -14,7 +14,7 @@ class LaunchTest(unittest.TestCase):
                                          "org.mozilla.fennec/.App -a "
                                          "android.intent.action.VIEW",
                                          "OK\nreturn code [0]")])
-        d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG)
         d.launchFennec("org.mozilla.fennec")
         a.wait()
 
@@ -27,7 +27,7 @@ class LaunchTest(unittest.TestCase):
                                          "org.mozilla.fennec/.App -a "
                                          "android.intent.action.VIEW",
                                          "OK\nreturn code [0]")])
-        d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG)
         d.launchFennec("org.mozilla.fennec")
         a.wait()
 

testing/mozbase/mozdevice/tests/sut_app.py

@@ -1,6 +1,6 @@
 #/usr/bin/env python
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -12,7 +12,7 @@ class TestApp(unittest.TestCase):
                     "/data/data/org.mozilla.firefox")]
 
         m = MockAgent(self, commands=command)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
 
         self.assertEqual(command[0][1], d.getAppRoot('org.mozilla.firefox'))
 

testing/mozbase/mozdevice/tests/sut_basic.py

@@ -1,6 +1,6 @@
 from sut import MockAgent
 import mozdevice
-import mozlog
+import logging
 import unittest
 
 class BasicTest(unittest.TestCase):
@@ -9,7 +9,7 @@ class BasicTest(unittest.TestCase):
         """Tests DeviceManager initialization."""
         a = MockAgent(self)
 
-        mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=mozlog.DEBUG)
+        mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG)
         # all testing done in device's constructor
         a.wait()
 
@@ -19,7 +19,7 @@ class BasicTest(unittest.TestCase):
         self.assertRaises(mozdevice.DMError,
                           lambda: mozdevice.DroidSUT("127.0.0.1",
                                                      port=a.port,
-                                                     logLevel=mozlog.DEBUG))
+                                                     logLevel=logging.DEBUG))
         a.wait()
 
     def test_timeout_normal(self):
@@ -29,7 +29,7 @@ class BasicTest(unittest.TestCase):
                                         ("ls", "test.txt"),
                                         ("rm /mnt/sdcard/tests/test.txt",
                                          "Removed the file")])
-        d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG)
         ret = d.removeFile('/mnt/sdcard/tests/test.txt')
         self.assertEqual(ret, None) # if we didn't throw an exception, we're ok
         a.wait()
@@ -40,7 +40,7 @@ class BasicTest(unittest.TestCase):
                                         ("cd /mnt/sdcard/tests", ""),
                                         ("ls", "test.txt"),
                                         ("rm /mnt/sdcard/tests/test.txt", 0)])
-        d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=a.port, logLevel=logging.DEBUG)
         d.default_timeout = 1
         exceptionThrown = False
         try:

testing/mozbase/mozdevice/tests/sut_chmod.py

@@ -1,6 +1,6 @@
 #/usr/bin/env python
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -13,7 +13,7 @@ class TestChmod(unittest.TestCase):
                                               '        <empty>\n'
                                               'chmod /storage/emulated/legacy/Test ok\n')]
         m = MockAgent(self, commands=command)
-        d = mozdevice.DroidSUT('127.0.0.1', port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT('127.0.0.1', port=m.port, logLevel=logging.DEBUG)
 
         self.assertEqual(None, d.chmodDir('/mnt/sdcard/test'))
 

testing/mozbase/mozdevice/tests/sut_copytree.py

@@ -6,7 +6,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -18,7 +18,7 @@ class CopyTreeTest(unittest.TestCase):
                     ('ls', 'test.txt\ntest2.txt')]
 
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
 
         self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/test.txt',
                 '/mnt/sdcard/tests/test2.txt'))
@@ -33,7 +33,7 @@ class CopyTreeTest(unittest.TestCase):
 
         m = MockAgent(self, commands=commands)
         d = mozdevice.DroidSUT("127.0.0.1", port=m.port,
-                logLevel=mozlog.DEBUG)
+                logLevel=logging.DEBUG)
 
         self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/foo',
                 '/mnt/sdcard/tests/bar'))
@@ -52,7 +52,7 @@ class CopyTreeTest(unittest.TestCase):
 
         m = MockAgent(self, commands=commands)
         d = mozdevice.DroidSUT("127.0.0.1", port=m.port,
-                logLevel=mozlog.DEBUG)
+                logLevel=logging.DEBUG)
 
         self.assertTrue(d.dirExists('/mnt/sdcard/tests/foo/bar'))
         self.assertEqual(None, d.copyTree('/mnt/sdcard/tests/foo',

testing/mozbase/mozdevice/tests/sut_fileMethods.py

@@ -2,7 +2,7 @@
 
 import hashlib
 import mozdevice
-import mozlog
+import logging
 import shutil
 import tempfile
 import unittest
@@ -27,14 +27,14 @@ class TestFileMethods(unittest.TestCase):
             commands_valid = [("hash /sdcard/test/file", self.temp_hash)]
 
             m = MockAgent(self, commands=commands_valid)
-            d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+            d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
             self.assertTrue(d.validateFile('/sdcard/test/file', f.name))
 
             # Test invalid hashes
             commands_invalid = [("hash /sdcard/test/file", "0this0hash0is0completely0invalid")]
 
             m = MockAgent(self, commands=commands_invalid)
-            d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+            d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
             self.assertFalse(d.validateFile('/sdcard/test/file', f.name))
 
     def test_getFile(self):
@@ -45,7 +45,7 @@ class TestFileMethods(unittest.TestCase):
 
         with tempfile.NamedTemporaryFile() as f:
             m = MockAgent(self, commands=commands)
-            d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+            d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
             # No error means success
             self.assertEqual(None, d.getFile(fname, f.name))
 
@@ -62,7 +62,7 @@ class TestFileMethods(unittest.TestCase):
 
         tmpdir = tempfile.mkdtemp()
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         self.assertEqual(None, d.getDirectory("/mnt/sdcard", tmpdir))
 
         # Cleanup

testing/mozbase/mozdevice/tests/sut_info.py

@@ -1,6 +1,6 @@
 #/usr/bin/env python
 import mozdevice
-import mozlog
+import logging
 import re
 import unittest
 from sut import MockAgent
@@ -36,7 +36,7 @@ class TestGetInfo(unittest.TestCase):
 
         for directive in self.commands.keys():
             m = MockAgent(self, commands=[self.commands[directive]])
-            d = mozdevice.DroidSUT('127.0.0.1', port=m.port, logLevel=mozlog.DEBUG)
+            d = mozdevice.DroidSUT('127.0.0.1', port=m.port, logLevel=logging.DEBUG)
 
             expected = re.sub(r'\ +', ' ', self.commands[directive][1]).split('\n')
             # Account for slightly different return format for 'process'

testing/mozbase/mozdevice/tests/sut_ip.py

@@ -1,6 +1,6 @@
 #/usr/bin/env python
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -20,17 +20,17 @@ class TestGetIP(unittest.TestCase):
 
     def test_getIP_eth0(self):
         m = MockAgent(self, commands=[self.commands[0]])
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         self.assertEqual('192.168.0.1', d.getIP(interfaces=['eth0']))
 
     def test_getIP_wlan0(self):
         m = MockAgent(self, commands=[self.commands[1]])
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         self.assertEqual('10.1.39.126', d.getIP(interfaces=['wlan0']))
 
     def test_getIP_error(self):
         m = MockAgent(self, commands=[self.commands[2]])
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         self.assertRaises(mozdevice.DMError, d.getIP, interfaces=['fake0'])
 
 if __name__ == '__main__':

testing/mozbase/mozdevice/tests/sut_kill.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -15,7 +15,7 @@ class TestKill(unittest.TestCase):
                     ("kill com.android.settings",
                      "Successfully killed com.android.settings\n")]
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         # No error raised means success
         self.assertEqual(None,  d.killProcess("com.android.settings"))
 

testing/mozbase/mozdevice/tests/sut_list.py

@@ -1,6 +1,6 @@
 #/usr/bin/env python
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -13,7 +13,7 @@ class TestListFiles(unittest.TestCase):
 
     def test_listFiles(self):
         m = MockAgent(self, commands=self.commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
 
         expected = (self.commands[2][1].strip()).split("\n")
         self.assertEqual(expected, d.listFiles("/mnt/sdcard"))

testing/mozbase/mozdevice/tests/sut_logcat.py

@@ -1,32 +1,32 @@
 #!/usr/bin/env python
 
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
 
 class TestLogCat(unittest.TestCase):
-    """ Class to test methods assosiated with logcat """
+    """ Class to test methods associated with logcat """
 
     def test_getLogcat(self):
 
-        logcat_output = ("07-17 00:51:10.377 I/SUTAgentAndroid( 2933): onCreate\n\r"
-        "07-17 00:51:10.457 D/dalvikvm( 2933): GC_CONCURRENT freed 351K, 17% free 2523K/3008K, paused 5ms+2ms, total 38ms\n\r"
-        "07-17 00:51:10.497 I/SUTAgentAndroid( 2933): Caught exception creating file in /data/local/tmp: open failed: EACCES (Permission denied)\n\r"
-        "07-17 00:51:10.507 E/SUTAgentAndroid( 2933): ERROR: Cannot access world writeable test root\n\r"
-        "07-17 00:51:10.547 D/GeckoHealthRec( 3253): Initializing profile cache.\n\r"
-        "07-17 00:51:10.607 D/GeckoHealthRec( 3253): Looking for /data/data/org.mozilla.fennec/files/mozilla/c09kfhne.default/times.json\n\r"
-        "07-17 00:51:10.637 D/GeckoHealthRec( 3253): Using times.json for profile creation time.\n\r"
-        "07-17 00:51:10.707 D/GeckoHealthRec( 3253): Incorporating environment: times.json profile creation = 1374026758604\n\r"
-        "07-17 00:51:10.507 D/GeckoHealthRec( 3253): Requested prefs.\n\r"
-        "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): \n\r"
-        "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Private Dirty Memory         3176 kb\n\r"
-        "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Proportional Set Size Memory 5679 kb\n\r"
-        "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Shared Dirty Memory          9216 kb\n\r"
+        logcat_output = ("07-17 00:51:10.377 I/SUTAgentAndroid( 2933): onCreate\r\n"
+        "07-17 00:51:10.457 D/dalvikvm( 2933): GC_CONCURRENT freed 351K, 17% free 2523K/3008K, paused 5ms+2ms, total 38ms\r\n"
+        "07-17 00:51:10.497 I/SUTAgentAndroid( 2933): Caught exception creating file in /data/local/tmp: open failed: EACCES (Permission denied)\r\n"
+        "07-17 00:51:10.507 E/SUTAgentAndroid( 2933): ERROR: Cannot access world writeable test root\r\n"
+        "07-17 00:51:10.547 D/GeckoHealthRec( 3253): Initializing profile cache.\r\n"
+        "07-17 00:51:10.607 D/GeckoHealthRec( 3253): Looking for /data/data/org.mozilla.fennec/files/mozilla/c09kfhne.default/times.json\r\n"
+        "07-17 00:51:10.637 D/GeckoHealthRec( 3253): Using times.json for profile creation time.\r\n"
+        "07-17 00:51:10.707 D/GeckoHealthRec( 3253): Incorporating environment: times.json profile creation = 1374026758604\r\n"
+        "07-17 00:51:10.507 D/GeckoHealthRec( 3253): Requested prefs.\r\n"
+        "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): \r\n"
+        "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Private Dirty Memory         3176 kb\r\n"
+        "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Proportional Set Size Memory 5679 kb\r\n"
+        "07-17 06:50:54.907 I/SUTAgentAndroid( 3876): Total Shared Dirty Memory          9216 kb\r\n"
         "07-17 06:55:21.627 I/SUTAgentAndroid( 3876): 127.0.0.1 : execsu /system/bin/logcat -v time -d dalvikvm:I "
-        "ConnectivityService:S WifiMonitor:S WifiStateTracker:S wpa_supplicant:S NetworkStateTracker:S\n\r"
-        "07-17 06:55:21.827 I/dalvikvm-heap( 3876): Grow heap (frag case) to 3.019MB for 102496-byte allocation\n\r"
+        "ConnectivityService:S WifiMonitor:S WifiStateTracker:S wpa_supplicant:S NetworkStateTracker:S\r\n"
+        "07-17 06:55:21.827 I/dalvikvm-heap( 3876): Grow heap (frag case) to 3.019MB for 102496-byte allocation\r\n"
         "return code [0]")
 
         inp = ("execsu /system/bin/logcat -v time -d "
@@ -35,15 +35,15 @@ class TestLogCat(unittest.TestCase):
 
         commands = [(inp, logcat_output)]
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
-        self.assertEqual(logcat_output[:-17].split('\r'), d.getLogcat())
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
+        self.assertEqual(logcat_output[:-17].replace('\r\n', '\n').splitlines(True), d.getLogcat())
 
     def test_recordLogcat(self):
 
         commands = [("execsu /system/bin/logcat -c", "return code [0]")]
 
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         # No error raised means success
         self.assertEqual(None, d.recordLogcat())
 

testing/mozbase/mozdevice/tests/sut_mkdir.py

@@ -2,7 +2,7 @@
 # http://creativecommons.org/publicdomain/zero/1.0/
 
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -35,7 +35,7 @@ class MkDirsTest(unittest.TestCase):
             exceptionThrown = False
             try:
                 d = mozdevice.DroidSUT('127.0.0.1', port=a.port,
-                                       logLevel=mozlog.DEBUG)
+                                       logLevel=logging.DEBUG)
                 d.mkDirs('/mnt/sdcard/baz/boop/bip')
             except mozdevice.DMError:
                 exceptionThrown = True
@@ -58,7 +58,7 @@ class MkDirsTest(unittest.TestCase):
                  '/mnt/sdcard/foo successfully created')]
         a = MockAgent(self, commands=cmds)
         d = mozdevice.DroidSUT('127.0.0.1', port=a.port,
-                               logLevel=mozlog.DEBUG)
+                               logLevel=logging.DEBUG)
         d.mkDirs('/mnt/sdcard/foo/foo')
         a.wait()
 
@@ -66,7 +66,7 @@ class MkDirsTest(unittest.TestCase):
         cmds = [('isdir /', 'TRUE')]
         a = MockAgent(self, commands=cmds)
         d = mozdevice.DroidSUT('127.0.0.1', port=a.port,
-                               logLevel=mozlog.DEBUG)
+                               logLevel=logging.DEBUG)
         d.mkDirs('/foo')
 
         a.wait()

testing/mozbase/mozdevice/tests/sut_movetree.py

@@ -6,7 +6,7 @@
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -21,7 +21,7 @@ class MoveTreeTest(unittest.TestCase):
                     ('ls', 'test1.txt')]
 
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/test.txt',
                 '/mnt/sdcard/tests/test1.txt'))
         self.assertFalse(d.fileExists('/mnt/sdcard/tests/test.txt'))
@@ -34,7 +34,7 @@ class MoveTreeTest(unittest.TestCase):
                     ('ls', 'bar')]
 
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/foo',
                 '/mnt/sdcard/tests/bar'))
         self.assertTrue(d.fileExists('/mnt/sdcard/tests/bar'))
@@ -51,7 +51,7 @@ class MoveTreeTest(unittest.TestCase):
 
         m = MockAgent(self, commands=commands)
         d = mozdevice.DroidSUT("127.0.0.1", port=m.port,
-                logLevel=mozlog.DEBUG)
+                logLevel=logging.DEBUG)
 
         self.assertTrue(d.dirExists('/mnt/sdcard/tests/foo/bar'))
         self.assertEqual(None, d.moveTree('/mnt/sdcard/tests/foo',

testing/mozbase/mozdevice/tests/sut_pull.py

@@ -1,6 +1,6 @@
 from sut import MockAgent
 import mozdevice
-import mozlog
+import logging
 import unittest
 
 class PullTest(unittest.TestCase):
@@ -20,7 +20,7 @@ class PullTest(unittest.TestCase):
                                             ("isdir /mnt/sdcard", "TRUE")])
 
             d = mozdevice.DroidSUT("127.0.0.1", port=a.port,
-                                   logLevel=mozlog.DEBUG)
+                                   logLevel=logging.DEBUG)
             pulledData = d.pullFile("/mnt/sdcard/cheeseburgers")
             self.assertEqual(pulledData, cheeseburgers)
             d.dirExists('/mnt/sdcard')
@@ -34,7 +34,7 @@ class PullTest(unittest.TestCase):
                                          "%s,15\n%s" % (remoteName,
                                                         "cheeseburgh"))])
         d = mozdevice.DroidSUT("127.0.0.1", port=a.port,
-                               logLevel=mozlog.DEBUG)
+                               logLevel=logging.DEBUG)
         exceptionThrown = False
         try:
             d.pullFile("/mnt/sdcard/cheeseburgers")

testing/mozbase/mozdevice/tests/sut_push.py

@@ -1,6 +1,7 @@
 from sut import MockAgent
+import mozfile
 import mozdevice
-import mozlog
+import logging
 import unittest
 import hashlib
 import tempfile
@@ -38,6 +39,7 @@ class PushTest(unittest.TestCase):
         expectedFileResponse = mdsum.hexdigest()
 
         tempdir = tempfile.mkdtemp()
+        self.addCleanup(mozfile.remove, tempdir)
         complex_path = os.path.join(tempdir, "baz")
         os.mkdir(complex_path)
         f = tempfile.NamedTemporaryFile(dir=complex_path)
@@ -71,7 +73,7 @@ class PushTest(unittest.TestCase):
             exceptionThrown = False
             try:
                 d = mozdevice.DroidSUT("127.0.0.1", port=a.port,
-                                       logLevel=mozlog.DEBUG)
+                                       logLevel=logging.DEBUG)
                 d.pushDir(tempdir, "/mnt/sdcard")
             except mozdevice.DMError:
                 exceptionThrown = True

testing/mozbase/mozdevice/tests/sut_remove.py

@@ -1,6 +1,6 @@
 #/usr/bin/env python
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -16,7 +16,7 @@ class TestRemove(unittest.TestCase):
                                             "/storage/emulated/legacy/Moztest\n")]
 
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         # No error implies we're all good
         self.assertEqual(None, d.removeDir("/mnt/sdcard/test"))
 

testing/mozbase/mozdevice/tests/sut_time.py

@@ -1,6 +1,6 @@
 #/usr/bin/env python
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -11,7 +11,7 @@ class TestGetCurrentTime(unittest.TestCase):
         command = [('clok', '1349980200')]
 
         m = MockAgent(self, commands=command)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         self.assertEqual(d.getCurrentTime(), int(command[0][1]))
 
 if __name__ == '__main__':

testing/mozbase/mozdevice/tests/sut_unpackfile.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 
 import mozdevice
-import mozlog
+import logging
 import unittest
 from sut import MockAgent
 
@@ -14,7 +14,7 @@ class TestUnpack(unittest.TestCase):
                      "Checksum:          653400271\n"
                      "1 of 1 successfully extracted\n")]
         m = MockAgent(self, commands=commands)
-        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=mozlog.DEBUG)
+        d = mozdevice.DroidSUT("127.0.0.1", port=m.port, logLevel=logging.DEBUG)
         # No error being thrown imples all is well
         self.assertEqual(None, d.unpackFile("/data/test/sample.zip",
                                             "/data/test/"))

testing/mozbase/mozhttpd/tests/api.py

@@ -4,6 +4,7 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
+import mozfile
 import mozhttpd
 import urllib2
 import os
@@ -194,6 +195,7 @@ class ProxyTest(unittest.TestCase):
 
     def test_proxy(self):
         docroot = tempfile.mkdtemp()
+        self.addCleanup(mozfile.remove, docroot)
         hosts = ('mozilla.com', 'mozilla.org')
         unproxied_host = 'notmozilla.org'
         def url(host): return 'http://%s/' % host

testing/mozbase/mozlog/mozlog/__init__.py

@@ -1,26 +1,25 @@
 # This Source Code Form is subject to the terms of the Mozilla Public
-# License, v. 2.0. If a copy of the MPL was not distributed with this file,
-# You can obtain one at http://mozilla.org/MPL/2.0/.
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 """
-Mozlog aims to standardize log formatting within Mozilla.
+Mozlog aims to standardize log handling and formatting within Mozilla.
 
-It simply wraps Python's logging_ module and adds a few convenience methods
-for logging test results and events.
+It implements a JSON-based structured logging protocol with convenience
+facilities for recording test results.
 
-The structured submodule takes a different approach and implements a
-JSON-based logging protocol designed for recording test results."""
+The old unstructured module is deprecated. It simply wraps Python's
+logging_ module and adds a few convenience methods for logging test
+results and events.
+"""
 
-from logger import *
-from loglistener import LogMessageServer
-from loggingmixin import LoggingMixin
+import sys
 
-try:
-    import structured
-except ImportError:
-    # Structured logging doesn't work on python 2.6 which is still used on some
-    # legacy test machines; https://bugzilla.mozilla.org/show_bug.cgi?id=864866
-    # Once we move away from Python 2.6, please cleanup devicemanager.py's
-    # exception block
-    pass
+from . import commandline
+from . import structuredlog
+from . import unstructured
+from .structuredlog import get_default_logger, set_default_logger
 
+# Backwards compatibility shim for consumers that use mozlog.structured
+structured = sys.modules[__name__]
+sys.modules['{}.structured'.format(__name__)] = structured

testing/mozbase/mozlog/mozlog/commandline.py

@@ -2,13 +2,15 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import sys
+import argparse
 import optparse
-
+import os
+import sys
 from collections import defaultdict
-from structuredlog import StructuredLogger, set_default_logger
-import handlers
-import formatters
+
+from . import handlers
+from . import formatters
+from .structuredlog import StructuredLogger, set_default_logger
 
 log_formatters = {
     'raw': (formatters.JSONFormatter, "Raw structured log messages"),
@@ -17,8 +19,12 @@ log_formatters = {
     'html': (formatters.HTMLFormatter, "HTML report"),
     'mach': (formatters.MachFormatter, "Human-readable output"),
     'tbpl': (formatters.TbplFormatter, "TBPL style log format"),
+    'errorsummary': (formatters.ErrorSummaryFormatter, argparse.SUPPRESS),
 }
 
+TEXT_FORMATTERS = ('raw', 'mach')
+"""a subset of formatters for non test harnesses related applications"""
+
 def level_filter_wrapper(formatter, level):
     return handlers.LogLevelFilter(formatter, level)
 
@@ -33,10 +39,20 @@ def buffer_handler_wrapper(handler, buffer_limit):
         buffer_limit = int(buffer_limit)
     return handlers.BufferingLogFilter(handler, buffer_limit)
 
-formatter_option_defaults = {
-    'verbose': False,
-    'level': 'info',
-}
+def default_formatter_options(log_type, overrides):
+    formatter_option_defaults = {
+        "raw": {
+            "level": "debug"
+        }
+    }
+    rv = {"verbose": False,
+          "level": "info"}
+    rv.update(formatter_option_defaults.get(log_type, {}))
+
+    if overrides is not None:
+        rv.update(overrides)
+
+    return rv
 
 fmt_options = {
     # <option name>: (<wrapper function>, description, <applicable formatters>, action)
@@ -46,7 +62,7 @@ fmt_options = {
                 ["mach"], "store_true"),
     'level': (level_filter_wrapper,
               "A least log level to subscribe to for the given formatter (debug, info, error, etc.)",
-              ["mach", "tbpl"], "store"),
+              ["mach", "raw", "tbpl"], "store"),
     'buffer': (buffer_handler_wrapper,
                "If specified, enables message buffering at the given buffer size limit.",
                ["mach", "tbpl"], "store"),
@@ -56,11 +72,14 @@ fmt_options = {
 def log_file(name):
     if name == "-":
         return sys.stdout
-    else:
-        return open(name, "w")
+    # ensure we have a correct dirpath by using realpath
+    dirpath = os.path.dirname(os.path.realpath(name))
+    if not os.path.exists(dirpath):
+        os.makedirs(dirpath)
+    return open(name, "w")
 
 
-def add_logging_group(parser):
+def add_logging_group(parser, include_formatters=None):
     """
     Add logging options to an argparse ArgumentParser or
     optparse OptionParser.
@@ -71,39 +90,46 @@ def add_logging_group(parser):
 
     :param parser: The ArgumentParser or OptionParser object that should have
                    logging options added.
+    :param include_formatters: List of formatter names that should be included
+                               in the option group. Default to None, meaning
+                               all the formatters are included. A common use
+                               of this option is to specify
+                               :data:`TEXT_FORMATTERS` to include only the
+                               most useful formatters for a command line tool
+                               that is not related to test harnesses.
     """
     group_name = "Output Logging"
     group_description = ("Each option represents a possible logging format "
                          "and takes a filename to write that format to, "
                          "or '-' to write to stdout.")
 
+    if include_formatters is None:
+        include_formatters = log_formatters.keys()
+
     if isinstance(parser, optparse.OptionParser):
         group = optparse.OptionGroup(parser,
                                      group_name,
                                      group_description)
-        for name, (cls, help_str) in log_formatters.iteritems():
-            group.add_option("--log-" + name, action="append", type="str",
-                             help=help_str)
-        for optname, (cls, help_str, formatters, action) in fmt_options.iteritems():
-            for fmt in formatters:
-                # make sure fmt wasn't removed from log_formatters
-                if fmt in log_formatters:
-                    group.add_option("--log-%s-%s" % (fmt, optname), action=action,
-                                     help=help_str, default=None)
         parser.add_option_group(group)
+        opt_log_type = 'str'
+        group_add = group.add_option
     else:
         group = parser.add_argument_group(group_name,
                                           group_description)
-        for name, (cls, help_str) in log_formatters.iteritems():
-            group.add_argument("--log-" + name, action="append", type=log_file,
-                               help=help_str)
+        opt_log_type = log_file
+        group_add = group.add_argument
 
-        for optname, (cls, help_str, formatters, action) in fmt_options.iteritems():
-            for fmt in formatters:
-                # make sure fmt wasn't removed from log_formatters
-                if fmt in log_formatters:
-                    group.add_argument("--log-%s-%s" % (fmt, optname), action=action,
-                                       help=help_str, default=None)
+    for name, (cls, help_str) in log_formatters.iteritems():
+        if name in include_formatters:
+            group_add("--log-" + name, action="append", type=opt_log_type,
+                      help=help_str)
+
+    for optname, (cls, help_str, formatters, action) in fmt_options.iteritems():
+        for fmt in formatters:
+            # make sure fmt is in log_formatters and is accepted
+            if fmt in log_formatters and fmt in include_formatters:
+                group_add("--log-%s-%s" % (fmt, optname), action=action,
+                          help=help_str, default=None)
 
 
 def setup_handlers(logger, formatters, formatter_options):
@@ -139,12 +165,12 @@ def setup_handlers(logger, formatters, formatter_options):
             logger.add_handler(handler)
 
 
-def setup_logging(suite, args, defaults=None):
+def setup_logging(suite, args, defaults=None, formatter_defaults=None):
     """
     Configure a structuredlogger based on command line arguments.
 
     The created structuredlogger will also be set as the default logger, and
-    can be retrieved with :py:func:`~mozlog.structured.structuredlog.get_default_logger`.
+    can be retrieved with :py:func:`~mozlog.get_default_logger`.
 
     :param suite: The name of the testsuite being run
     :param args: A dictionary of {argument_name:value} produced from
@@ -154,13 +180,14 @@ def setup_logging(suite, args, defaults=None):
                      this isn't supplied, reasonable defaults are chosen
                      (coloured mach formatting if stdout is a terminal, or raw
                      logs otherwise).
-
+    :param formatter_defaults: A dictionary of {option_name: default_value} to provide
+                               to the formatters in the absence of command line overrides.
     :rtype: StructuredLogger
     """
 
     logger = StructuredLogger(suite)
     # Keep track of any options passed for formatters.
-    formatter_options = defaultdict(lambda: formatter_option_defaults.copy())
+    formatter_options = {}
     # Keep track of formatters and list of streams specified.
     formatters = defaultdict(list)
     found = False
@@ -195,6 +222,9 @@ def setup_logging(suite, args, defaults=None):
                     formatters[formatter].append(value)
             if len(parts) == 3:
                 _, formatter, opt = parts
+                if formatter not in formatter_options:
+                    formatter_options[formatter] = default_formatter_options(formatter,
+                                                                             formatter_defaults)
                 formatter_options[formatter][opt] = values
 
     #If there is no user-specified logging, go with the default options
@@ -207,6 +237,10 @@ def setup_logging(suite, args, defaults=None):
             if value == sys.stdout:
                 formatters[name].append(value)
 
+    for name in formatters:
+        if name not in formatter_options:
+            formatter_options[name] = default_formatter_options(name, formatter_defaults)
+
     setup_handlers(logger, formatters, formatter_options)
     set_default_logger(logger)
 

testing/mozbase/mozlog/mozlog/formatters/__init__.py

@@ -8,6 +8,7 @@ from xunit import XUnitFormatter
 from html import HTMLFormatter
 from machformatter import MachFormatter
 from tbplformatter import TbplFormatter
+from errorsummary import ErrorSummaryFormatter
 
 def JSONFormatter():
     return lambda x: json.dumps(x) + "\n"

testing/mozbase/mozlog/mozlog/formatters/errorsummary.py

@@ -0,0 +1,55 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+import json
+
+from base import BaseFormatter
+
+class ErrorSummaryFormatter(BaseFormatter):
+    def __init__(self):
+        self.line_count = 0
+
+    def __call__(self, data):
+        rv = BaseFormatter.__call__(self, data)
+        self.line_count += 1
+        return rv
+
+    def _output(self, data_type, data):
+        data["action"] = data_type
+        data["line"] = self.line_count
+        return "%s\n" % json.dumps(data)
+
+    def _output_test(self, test, subtest, item):
+        data = {"test": test,
+                "subtest": subtest,
+                "status": item["status"],
+                "expected": item["expected"],
+                "message": item.get("message"),
+                "stack": item.get("stack")}
+        return self._output("test_result", data)
+
+    def test_status(self, item):
+        if "expected" not in item:
+            return
+        return self._output_test(item["test"], item["subtest"], item)
+
+    def test_end(self, item):
+        if "expected" not in item:
+            return
+        return self._output_test(item["test"], None, item)
+
+    def log(self, item):
+        if item["level"] not in ("ERROR", "CRITICAL"):
+            return
+
+        data = {"level": item["level"],
+                "message": item["message"]}
+        return self._output("log", data)
+
+    def crash(self, item):
+        data = {"test": item.get("test"),
+                "signature": item["signature"],
+                "stackwalk_stdout": item.get("stackwalk_stdout"),
+                "stackwalk_stderr": item.get("stackwalk_stderr")}
+        return self._output("crash", data)

testing/mozbase/mozlog/mozlog/handlers/statushandler.py

@@ -7,7 +7,7 @@ from collections import (
     namedtuple,
 )
 
-from mozlog.structured.structuredlog import log_levels
+from mozlog.structuredlog import log_levels
 
 RunSummary = namedtuple("RunSummary",
                         ("unexpected_statuses",

testing/mozbase/mozlog/mozlog/scripts/logmerge.py

@@ -5,7 +5,7 @@ import os
 import sys
 from threading import current_thread
 import time
-from mozlog.structured.reader import read
+from mozlog.reader import read
 
 
 def dump_entry(entry, output):
@@ -79,4 +79,4 @@ if __name__ == "__main__":
     parser = get_parser()
     args = parser.parse_args()
     kwargs = vars(args)
-    main(**kwargs)
+    main(**kwargs)

testing/mozbase/mozlog/mozlog/scripts/unstable.py

@@ -2,7 +2,7 @@ import argparse
 from collections import defaultdict
 import json
 
-from mozlog.structured import reader
+from mozlog import reader
 
 class StatusHandler(reader.LogHandler):
     def __init__(self):

testing/mozbase/mozlog/mozlog/structuredlog.py

@@ -80,7 +80,7 @@ def set_default_logger(default_logger):
 
     It can then be retrieved with :py:func:`get_default_logger`
 
-    Note that :py:func:`~mozlog.structured.commandline.setup_logging` will
+    Note that :py:func:`~mozlog.commandline.setup_logging` will
     set a default logger for you, so there should be no need to call this
     function if you're using setting up logging that way (recommended).
 

testing/mozbase/mozlog/mozlog/unstructured/__init__.py

@@ -2,6 +2,6 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import commandline
-import structuredlog
-from structuredlog import get_default_logger, set_default_logger
+from .logger import *
+from .loglistener import LogMessageServer
+from .loggingmixin import LoggingMixin

testing/mozbase/mozlog/mozlog/unstructured/loggingmixin.py

@@ -2,7 +2,11 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import mozlog
+from .logger import (
+    Logger,
+    getLogger,
+)
+
 
 class LoggingMixin(object):
     """Expose a subset of logging functions to an inheriting class."""
@@ -10,19 +14,18 @@ class LoggingMixin(object):
     def set_logger(self, logger_instance=None, name=None):
         """Method for setting the underlying logger instance to be used."""
 
-        if logger_instance and not isinstance(logger_instance, mozlog.Logger):
-            raise ValueError("logger_instance must be an instance of" +
-                             "mozlog.Logger")
+        if logger_instance and not isinstance(logger_instance, Logger):
+            raise ValueError("logger_instance must be an instance of Logger")
 
         if name is None:
             name = ".".join([self.__module__, self.__class__.__name__])
 
-        self._logger = logger_instance or mozlog.getLogger(name)
+        self._logger = logger_instance or getLogger(name)
 
     def _log_msg(self, cmd, *args, **kwargs):
         if not hasattr(self, "_logger"):
-            self._logger = mozlog.getLogger(".".join([self.__module__,
-                                                      self.__class__.__name__]))
+            self._logger = getLogger(".".join([self.__module__,
+                                               self.__class__.__name__]))
         getattr(self._logger, cmd)(*args, **kwargs)
 
     def log(self, *args, **kwargs):

testing/mozbase/mozlog/setup.py

@@ -5,7 +5,7 @@
 from setuptools import setup, find_packages
 
 PACKAGE_NAME = 'mozlog'
-PACKAGE_VERSION = '2.10'
+PACKAGE_VERSION = '3.0'
 
 setup(name=PACKAGE_NAME,
       version=PACKAGE_VERSION,
@@ -27,10 +27,10 @@ setup(name=PACKAGE_NAME,
                    'Operating System :: OS Independent',
                    'Topic :: Software Development :: Libraries :: Python Modules',
                   ],
-      package_data={"mozlog.structured": ["formatters/html/main.js",
-                                          "formatters/html/style.css"]},
+      package_data={"mozlog": ["formatters/html/main.js",
+                               "formatters/html/style.css"]},
       entry_points={
           "console_scripts": [
-              "structlog = mozlog.structured.scripts:main"
+              "structlog = mozlog.scripts:main"
           ]}
      )

testing/mozbase/mozlog/tests/test_logger.py

@@ -11,7 +11,7 @@ import unittest
 
 import mozfile
 
-import mozlog
+import mozlog.unstructured as mozlog
 
 class ListHandler(mozlog.Handler):
     """Mock handler appends messages to a list for later inspection."""

testing/mozbase/mozlog/tests/test_structured.py

@@ -10,7 +10,7 @@ import xml.etree.ElementTree as ET
 
 import mozfile
 
-from mozlog.structured import (
+from mozlog import (
     commandline,
     reader,
     structuredlog,
@@ -734,6 +734,22 @@ class TestCommandline(unittest.TestCase):
         self.assertEqual(len(logger.handlers), 1)
         self.assertIsInstance(logger.handlers[0], handlers.StreamHandler)
 
+    def test_limit_formatters(self):
+        parser = argparse.ArgumentParser()
+        commandline.add_logging_group(parser, include_formatters=['raw'])
+        other_formatters = [fmt for fmt in commandline.log_formatters
+                            if fmt != 'raw']
+        # check that every formatter except raw is not present
+        for fmt in other_formatters:
+            with self.assertRaises(SystemExit):
+                parser.parse_args(["--log-%s=-" % fmt])
+            with self.assertRaises(SystemExit):
+                parser.parse_args(["--log-%s-level=error" % fmt])
+        # raw is still ok
+        args = parser.parse_args(["--log-raw=-"])
+        logger = commandline.setup_logging("test_setup_logging2", args, {})
+        self.assertEqual(len(logger.handlers), 1)
+
     def test_setup_logging_optparse_unicode(self):
         parser = optparse.OptionParser()
         commandline.add_logging_group(parser)

testing/mozbase/moznetwork/moznetwork/moznetwork.py

@@ -2,23 +2,37 @@
 # License, v. 2.0. If a copy of the MPL was not distributed with this file,
 # You can obtain one at http://mozilla.org/MPL/2.0/.
 
-import socket
+import argparse
 import array
 import re
+import socket
 import struct
 import subprocess
+import sys
+
 import mozinfo
+import mozlog
 
 if mozinfo.isLinux:
     import fcntl
 
+
 class NetworkError(Exception):
     """Exception thrown when unable to obtain interface or IP."""
 
 
+def _get_logger():
+    logger = mozlog.get_default_logger(component='moznetwork')
+    if not logger:
+        logger = mozlog.unstructured.getLogger('moznetwork')
+    return logger
+
+
 def _get_interface_list():
     """Provides a list of available network interfaces
        as a list of tuples (name, ip)"""
+    logger = _get_logger()
+    logger.debug('Gathering interface list')
     max_iface = 32  # Maximum number of interfaces(Aribtrary)
     bytes = max_iface * 32
     is_32bit = (8 * struct.calcsize("P")) == 32  # Set Architecture
@@ -34,12 +48,13 @@ def _get_interface_list():
         ))[0]
         namestr = names.tostring()
         return [(namestr[i:i + 32].split('\0', 1)[0],
-                socket.inet_ntoa(namestr[i + 20:i + 24]))\
+                socket.inet_ntoa(namestr[i + 20:i + 24]))
                 for i in range(0, outbytes, struct_size)]
 
     except IOError:
         raise NetworkError('Unable to call ioctl with SIOCGIFCONF')
 
+
 def _proc_matches(args, regex):
     """Helper returns the matches of regex in the output of a process created with
     the given arguments"""
@@ -48,25 +63,34 @@ def _proc_matches(args, regex):
                               stderr=subprocess.STDOUT).stdout.read()
     return re.findall(regex, output)
 
+
 def _parse_ifconfig():
     """Parse the output of running ifconfig on mac in cases other methods
     have failed"""
+    logger = _get_logger()
+    logger.debug('Parsing ifconfig')
 
     # Attempt to determine the default interface in use.
     default_iface = _proc_matches(['route', '-n', 'get', 'default'],
                                   'interface: (\w+)')
+
     if default_iface:
         addr_list = _proc_matches(['ifconfig', default_iface[0]],
                                   'inet (\d+.\d+.\d+.\d+)')
-        if addr_list and not addr_list[0].startswith('127.'):
-            return addr_list[0]
+        if addr_list:
+            logger.debug('Default interface: [%s] %s' % (default_iface[0],
+                                                         addr_list[0]))
+            if not addr_list[0].startswith('127.'):
+                return addr_list[0]
 
     # Iterate over plausible interfaces if we didn't find a suitable default.
     for iface in ['en%s' % i for i in range(10)]:
         addr_list = _proc_matches(['ifconfig', iface],
                                   'inet (\d+.\d+.\d+.\d+)')
-        if addr_list and not addr_list[0].startswith('127.'):
-            return addr_list[0]
+        if addr_list:
+            logger.debug('Interface: [%s] %s' % (iface, addr_list[0]))
+            if not addr_list[0].startswith('127.'):
+                return addr_list[0]
 
     # Just return any that isn't localhost. If we can't find one, we have
     # failed.
@@ -77,16 +101,30 @@ def _parse_ifconfig():
     except IndexError:
         return None
 
+
 def get_ip():
     """Provides an available network interface address, for example
        "192.168.1.3".
 
        A `NetworkError` exception is raised in case of failure."""
+    logger = _get_logger()
     try:
+        hostname = socket.gethostname()
         try:
-            ip = socket.gethostbyname(socket.gethostname())
+            logger.debug('Retrieving IP for %s' % hostname)
+            ips = socket.gethostbyname_ex(hostname)[2]
         except socket.gaierror:  # for Mac OS X
-            ip = socket.gethostbyname(socket.gethostname() + ".local")
+            hostname += '.local'
+            logger.debug('Retrieving IP for %s' % hostname)
+            ips = socket.gethostbyname_ex(hostname)[2]
+        if len(ips) == 1:
+            ip = ips[0]
+        elif len(ips) > 1:
+            logger.debug('Multiple addresses found: %s' % ips)
+            # no fallback on Windows so take the first address
+            ip = ips[0] if mozinfo.isWin else None
+        else:
+            ip = None
     except socket.gaierror:
         # sometimes the hostname doesn't resolve to an ip address, in which
         # case this will always fail
@@ -96,6 +134,7 @@ def get_ip():
         if mozinfo.isLinux:
             interfaces = _get_interface_list()
             for ifconfig in interfaces:
+                logger.debug('Interface: [%s] %s' % (ifconfig[0], ifconfig[1]))
                 if ifconfig[0] == 'lo':
                     continue
                 else:
@@ -112,3 +151,22 @@ def get_ip():
 def get_lan_ip():
     """Deprecated. Please use get_ip() instead."""
     return get_ip()
+
+
+def cli(args=sys.argv[1:]):
+    parser = argparse.ArgumentParser(
+        description='Retrieve IP address')
+    structured.commandline.add_logging_group(
+        parser,
+        include_formatters=structured.commandline.TEXT_FORMATTERS
+    )
+
+    args = parser.parse_args()
+    structured.commandline.setup_logging(
+        'mozversion', args, {'mach': sys.stdout})
+
+    _get_logger().info('IP address: %s' % get_ip())
+
+
+if __name__ == '__main__':
+    cli()

testing/mozbase/moznetwork/setup.py

@@ -4,9 +4,11 @@
 
 from setuptools import setup
 
-PACKAGE_VERSION = '0.24'
+PACKAGE_VERSION = '0.26'
 
-deps=[ 'mozinfo' ]
+deps = ['mozinfo',
+        'mozlog >= 3.0',
+        ]
 
 setup(name='moznetwork',
       version=PACKAGE_VERSION,
@@ -21,5 +23,7 @@ setup(name='moznetwork',
       packages=['moznetwork'],
       include_package_data=True,
       zip_safe=False,
-      install_requires=deps
+      install_requires=deps,
+      entry_points={'console_scripts': [
+          'moznetwork = moznetwork:cli']},
       )

testing/mozbase/mozprofile/mozprofile/addons.py

@@ -10,15 +10,15 @@ import urllib2
 import zipfile
 from xml.dom import minidom
 
-from manifestparser import ManifestParser
 import mozfile
-import mozlog
+from manifestparser import ManifestParser
+from mozlog.unstructured import getLogger
 
 # Needed for the AMO's rest API - https://developer.mozilla.org/en/addons.mozilla.org_%28AMO%29_API_Developers%27_Guide/The_generic_AMO_API
 AMO_API_VERSION = "1.5"
 
 # Logger for 'mozprofile.addons' module
-module_logger = mozlog.getLogger(__name__)
+module_logger = getLogger(__name__)
 
 
 class AddonFormatError(Exception):