diff --git a/b2g/chrome/content/shell.js b/b2g/chrome/content/shell.js index dd17dc9285..7e357691a9 100644 --- a/b2g/chrome/content/shell.js +++ b/b2g/chrome/content/shell.js @@ -77,6 +77,11 @@ XPCOMUtils.defineLazyServiceGetter(Services, 'captivePortalDetector', '@mozilla.org/toolkit/captive-detector;1', 'nsICaptivePortalDetector'); +if (AppConstants.MOZ_SAFE_BROWSING) { + XPCOMUtils.defineLazyModuleGetter(this, "SafeBrowsing", + "resource://gre/modules/SafeBrowsing.jsm"); +} + XPCOMUtils.defineLazyModuleGetter(this, "SafeMode", "resource://gre/modules/SafeMode.jsm"); @@ -387,6 +392,12 @@ var shell = { ppmm.addMessageListener("sms-handler", this); ppmm.addMessageListener("mail-handler", this); ppmm.addMessageListener("file-picker", this); + + if (AppConstants.MOZ_SAFE_BROWSING) { + setTimeout(function() { + SafeBrowsing.init(); + }, 5000); + } }, stop: function shell_stop() { diff --git a/b2g/confvars.sh b/b2g/confvars.sh index 6546e8fed8..1dc8b45348 100755 --- a/b2g/confvars.sh +++ b/b2g/confvars.sh @@ -17,8 +17,8 @@ MOZ_BRANDING_DIRECTORY=b2g/branding/unofficial MOZ_OFFICIAL_BRANDING_DIRECTORY=b2g/branding/official # MOZ_APP_DISPLAYNAME is set by branding/configure.sh +MOZ_SAFE_BROWSING=1 MOZ_SERVICES_COMMON=1 -MOZ_SERVICES_METRICS=1 MOZ_WEBSMS_BACKEND=1 MOZ_NO_SMART_CARDS=1 diff --git a/browser/app/profile/palemoon.js b/browser/app/profile/palemoon.js index f8e01efde1..41f67ff3ff 100644 --- a/browser/app/profile/palemoon.js +++ b/browser/app/profile/palemoon.js @@ -390,6 +390,12 @@ pref("browser.search.suggest.enabled", true); pref("browser.search.official", true); #endif +#ifdef XP_WIN +pref("browser.search.redirectWindowsSearch", true); +#else +pref("browser.search.redirectWindowsSearch", false); +#endif + pref("browser.sessionhistory.max_entries", 50); // Built-in default permissions. diff --git a/browser/components/nsBrowserContentHandler.js b/browser/components/nsBrowserContentHandler.js index e534e12069..4bc74cf164 100644 --- a/browser/components/nsBrowserContentHandler.js +++ b/browser/components/nsBrowserContentHandler.js @@ -4,6 +4,7 @@ Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); Components.utils.import("resource://gre/modules/Services.jsm"); +Components.utils.import("resource://gre/modules/AppConstants.jsm"); XPCOMUtils.defineLazyModuleGetter(this, "PrivateBrowsingUtils", "resource://gre/modules/PrivateBrowsingUtils.jsm"); @@ -250,7 +251,7 @@ function doSearch(searchTerm, cmdLine) { var ss = Components.classes["@mozilla.org/browser/search-service;1"] .getService(nsIBrowserSearchService); - var submission = ss.defaultEngine.getSubmission(searchTerm); + var submission = ss.defaultEngine.getSubmission(searchTerm, null, "system"); // fill our nsISupportsArray with uri-as-wstring, null, null, postData var sa = Components.classes["@mozilla.org/supports-array;1"] @@ -666,10 +667,42 @@ nsDefaultCommandLineHandler.prototype = { } #endif + let redirectWinSearch = false; + if (AppConstants.isPlatformAndVersionAtLeast("win", "10")) { + redirectWinSearch = Services.prefs.getBoolPref("browser.search.redirectWindowsSearch"); + } + try { var ar; while ((ar = cmdLine.handleFlagWithParam("url", false))) { var uri = resolveURIInternal(cmdLine, ar); + + // Searches in the Windows 10 task bar searchbox simply open the default browser + // with a URL for a search on Bing. Here we extract the search term and use the + // user's default search engine instead. + if (redirectWinSearch && uri.spec.startsWith("https://www.bing.com/search")) { + try { + var url = uri.QueryInterface(Components.interfaces.nsIURL); + var params = new URLSearchParams(url.query); + // We don't want to rewrite all Bing URLs coming from external apps. Look + // for the magic URL parm that's present in searches from the task bar. + // (Typed searches use "form=WNSGPH", Cortana voice searches use "FORM=WNSBOX") + var formParam = params.get("form"); + if (!formParam) { + formParam = params.get("FORM"); + } + if (formParam == "WNSGPH" || formParam == "WNSBOX") { + var term = params.get("q"); + var ss = Components.classes["@mozilla.org/browser/search-service;1"] + .getService(nsIBrowserSearchService); + var submission = ss.defaultEngine.getSubmission(term, null, "system"); + uri = submission.uri; + } + } catch (e) { + Components.utils.reportError("Couldn't redirect Windows search: " + e); + } + } + urilist.push(uri); } } diff --git a/browser/components/preferences/in-content/search.xul b/browser/components/preferences/in-content/search.xul new file mode 100644 index 0000000000..08e646c70c --- /dev/null +++ b/browser/components/preferences/in-content/search.xul @@ -0,0 +1,79 @@ + + + + + + + + + + + + + diff --git a/dom/animation/test/css-animations/test_animation-player-currenttime.html b/dom/animation/test/css-animations/test_animation-player-currenttime.html new file mode 100644 index 0000000000..db237a726a --- /dev/null +++ b/dom/animation/test/css-animations/test_animation-player-currenttime.html @@ -0,0 +1,657 @@ + + + + + Tests for the effect of setting a CSS animation's + Animation.currentTime + + + + + + +
+ + + diff --git a/dom/animation/test/css-animations/test_animation-player-finished.html b/dom/animation/test/css-animations/test_animation-player-finished.html new file mode 100644 index 0000000000..95e5cea450 --- /dev/null +++ b/dom/animation/test/css-animations/test_animation-player-finished.html @@ -0,0 +1,290 @@ + + + + + +
+ + diff --git a/dom/animation/test/css-animations/test_animation-player-starttime.html b/dom/animation/test/css-animations/test_animation-player-starttime.html new file mode 100644 index 0000000000..57d277d231 --- /dev/null +++ b/dom/animation/test/css-animations/test_animation-player-starttime.html @@ -0,0 +1,680 @@ + + + + + Tests for the effect of setting a CSS animation's + Animation.startTime + + + + + + +
+ + + diff --git a/dom/animation/test/testcommon.js b/dom/animation/test/testcommon.js index b021bb219e..54add06692 100644 --- a/dom/animation/test/testcommon.js +++ b/dom/animation/test/testcommon.js @@ -38,6 +38,29 @@ function waitForFrame() { }); } +/** + * Returns a Promise that is resolved after the given number of consecutive + * animation frames have occured (using requestAnimationFrame callbacks). + * + * @param frameCount The number of animation frames. + * @param onFrame An optional function to be processed in each animation frame. + */ +function waitForAnimationFrames(frameCount, onFrame) { + return new Promise(function(resolve, reject) { + function handleFrame() { + if (--frameCount <= 0) { + resolve(); + } else { + if (onFrame && typeof onFrame === 'function') { + onFrame(); + } + window.requestAnimationFrame(handleFrame); // wait another frame + } + } + window.requestAnimationFrame(handleFrame); + }); +} + /** * Wrapper that takes a sequence of N animations and returns: * diff --git a/dom/network/NetworkStatsDB.jsm b/dom/network/NetworkStatsDB.jsm index 0ca075a0d8..aa74d40ad9 100644 --- a/dom/network/NetworkStatsDB.jsm +++ b/dom/network/NetworkStatsDB.jsm @@ -948,9 +948,20 @@ NetworkStatsDB.prototype = { let request = aStore.openCursor(range).onsuccess = function(event) { var cursor = event.target.result; if (cursor){ - data.push({ rxBytes: cursor.value.rxBytes, - txBytes: cursor.value.txBytes, - date: new Date(cursor.value.timestamp + offset) }); + // We use rxTotalBytes/txTotalBytes instead of rxBytes/txBytes for + // the first (oldest) sample. The rx/txTotalBytes fields record + // accumulative usage amount, which means even if old samples were + // expired and removed from the Database, we can still obtain the + // correct network usage. + if (data.length == 0) { + data.push({ rxBytes: cursor.value.rxTotalBytes, + txBytes: cursor.value.txTotalBytes, + date: new Date(cursor.value.timestamp + offset) }); + } else { + data.push({ rxBytes: cursor.value.rxBytes, + txBytes: cursor.value.txBytes, + date: new Date(cursor.value.timestamp + offset) }); + } cursor.continue(); return; } @@ -981,9 +992,20 @@ NetworkStatsDB.prototype = { foundData.rxBytes += cursor.value.rxBytes; foundData.txBytes += cursor.value.txBytes; } else { - data.push({ rxBytes: cursor.value.rxBytes, - txBytes: cursor.value.txBytes, - date: new Date(cursor.value.timestamp + offset) }); + // We use rxTotalBytes/txTotalBytes instead of rxBytes/txBytes + // for the first (oldest) sample. The rx/txTotalBytes fields + // record accumulative usage amount, which means even if old + // samples were expired and removed from the Database, we can + // still obtain the correct network usage. + if (data.length == 0) { + data.push({ rxBytes: cursor.value.rxTotalBytes, + txBytes: cursor.value.txTotalBytes, + date: new Date(cursor.value.timestamp + offset) }); + } else { + data.push({ rxBytes: cursor.value.rxBytes, + txBytes: cursor.value.txBytes, + date: new Date(cursor.value.timestamp + offset) }); + } } cursor.continue(); return; diff --git a/dom/network/tests/unit_stats/test_networkstats_db.js b/dom/network/tests/unit_stats/test_networkstats_db.js index ef4669ab3f..15e5b2c36e 100644 --- a/dom/network/tests/unit_stats/test_networkstats_db.js +++ b/dom/network/tests/unit_stats/test_networkstats_db.js @@ -797,22 +797,26 @@ add_test(function test_findBrowsingTrafficStats() { stats.push({ appId: 1008, isInBrowser: 0, serviceType: serviceType, network: networkMobile, timestamp: saveDate + (sampleRate * i), - rxBytes: 200, txBytes: 100}); + rxBytes: 200, txBytes: 100, + rxTotalBytes: 200, txTotalBytes: 100}); // Browser of system app. stats.push({ appId: 1008, isInBrowser: 1, serviceType: serviceType, network: networkMobile, timestamp: saveDate + (sampleRate * i), - rxBytes: 1000, txBytes: 500}); + rxBytes: 1000, txBytes: 500, + rxTotalBytes: 1000, txTotalBytes: 500}); // Another app. stats.push({ appId: 1021, isInBrowser: 0, serviceType: serviceType, network: networkMobile, timestamp: saveDate + (sampleRate * i), - rxBytes: 300, txBytes: 150}); + rxBytes: 300, txBytes: 150, + rxTotalBytes: 300, txTotalBytes: 150}); // Browser of another app. stats.push({ appId: 1021, isInBrowser: 1, serviceType: serviceType, network: networkMobile, timestamp: saveDate + (sampleRate * i), - rxBytes: 600, txBytes: 300}); + rxBytes: 600, txBytes: 300, + rxTotalBytes: 600, txTotalBytes: 300}); } prepareFind(stats, function(error, result) { @@ -853,22 +857,26 @@ add_test(function test_findAppTrafficStats() { stats.push({ appId: 1008, isInBrowser: 0, serviceType: serviceType, network: networkMobile, timestamp: saveDate + (sampleRate * i), - rxBytes: 200, txBytes: 100}); + rxBytes: 200, txBytes: 100, + rxTotalBytes: 200, txTotalBytes: 100}); // Browser of system app. stats.push({ appId: 1008, isInBrowser: 1, serviceType: serviceType, network: networkMobile, timestamp: saveDate + (sampleRate * i), - rxBytes: 1000, txBytes: 500}); + rxBytes: 1000, txBytes: 500, + rxTotalBytes: 1000, txTotalBytes: 500}); // Another app. stats.push({ appId: 1021, isInBrowser: 0, serviceType: serviceType, network: networkMobile, timestamp: saveDate + (sampleRate * i), - rxBytes: 300, txBytes: 150}); + rxBytes: 300, txBytes: 150, + rxTotalBytes: 300, txTotalBytes: 150}); // Browser of another app. stats.push({ appId: 1021, isInBrowser: 1, serviceType: serviceType, network: networkMobile, timestamp: saveDate + (sampleRate * i), - rxBytes: 600, txBytes: 300}); + rxBytes: 600, txBytes: 300, + rxTotalBytes: 600, txTotalBytes: 300}); } prepareFind(stats, function(error, result) { diff --git a/dom/promise/tests/unit/test_monitor_uncaught.js b/dom/promise/tests/unit/test_monitor_uncaught.js index e03448eb50..7dd80d212c 100644 --- a/dom/promise/tests/unit/test_monitor_uncaught.js +++ b/dom/promise/tests/unit/test_monitor_uncaught.js @@ -4,9 +4,13 @@ "use strict"; -const { utils: Cu } = Components; +var { utils: Cu } = Components; Cu.import("resource://gre/modules/Timer.jsm", this); +Cu.import("resource://testing-common/PromiseTestUtils.jsm", this); + +// Prevent test failures due to the unhandled rejections in this test file. +PromiseTestUtils.disableUncaughtRejectionObserverForSelfTest(); add_task(function* test_globals() { Assert.equal(Promise.defer || undefined, undefined, "We are testing DOM Promise."); diff --git a/dom/push/PushService.jsm b/dom/push/PushService.jsm index eb66178fe2..ef7b38e571 100644 --- a/dom/push/PushService.jsm +++ b/dom/push/PushService.jsm @@ -458,7 +458,7 @@ this.PushService = { // Before completing the activation check prefs. This will first check // connection.enabled pref and then check offline state. this._changeStateConnectionEnabledEvent(prefs.get("connection.enabled")); - }); + }).catch(Cu.reportError); } else { // This is only used for testing. Different tests require connecting to diff --git a/dom/push/PushServiceHttp2.jsm b/dom/push/PushServiceHttp2.jsm index 8d74f98e1c..949bee6131 100644 --- a/dom/push/PushServiceHttp2.jsm +++ b/dom/push/PushServiceHttp2.jsm @@ -734,12 +734,15 @@ this.PushServiceHttp2 = { .then(record => this._subscribeResource(record) .then(recordNew => { if (this._mainPushService) { - this._mainPushService.updateRegistrationAndNotifyApp(aSubscriptionUri, - recordNew); + this._mainPushService + .updateRegistrationAndNotifyApp(aSubscriptionUri, recordNew) + .catch(Cu.reportError); } }, error => { if (this._mainPushService) { - this._mainPushService.dropRegistrationAndNotifyApp(aSubscriptionUri); + this._mainPushService + .dropRegistrationAndNotifyApp(aSubscriptionUri) + .catch(Cu.reportError); } }) ); diff --git a/dom/push/test/xpcshell/test_registration_success_http2.js b/dom/push/test/xpcshell/test_registration_success_http2.js index e7d3c897f3..f955ba2688 100644 --- a/dom/push/test/xpcshell/test_registration_success_http2.js +++ b/dom/push/test/xpcshell/test_registration_success_http2.js @@ -4,6 +4,15 @@ 'use strict'; Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://testing-common/PromiseTestUtils.jsm"); + +/////////////////// +// +// Whitelisting this test. +// As part of bug 1077403, the leaking uncaught rejection should be fixed. +// +// Instances of the rejection "record is undefined" may or may not appear. +PromiseTestUtils.thisTestLeaksUncaughtRejectionsAndShouldBeFixed(); const {PushDB, PushService, PushServiceHttp2} = serviceExports; diff --git a/dom/push/test/xpcshell/test_unregister_success_http2.js b/dom/push/test/xpcshell/test_unregister_success_http2.js index b6c77ca3d6..c7c63f8293 100644 --- a/dom/push/test/xpcshell/test_unregister_success_http2.js +++ b/dom/push/test/xpcshell/test_unregister_success_http2.js @@ -4,6 +4,15 @@ 'use strict'; Cu.import("resource://gre/modules/Services.jsm"); +Cu.import("resource://testing-common/PromiseTestUtils.jsm"); + +/////////////////// +// +// Whitelisting this test. +// As part of bug 1077403, the leaking uncaught rejection should be fixed. +// +// Instances of the rejection "record is undefined" may or may not appear. +PromiseTestUtils.thisTestLeaksUncaughtRejectionsAndShouldBeFixed(); const {PushDB, PushService, PushServiceHttp2} = serviceExports; diff --git a/js/xpconnect/tests/unit/head_watchdog.js b/js/xpconnect/tests/unit/head_watchdog.js index e091c02b60..cf006303aa 100644 --- a/js/xpconnect/tests/unit/head_watchdog.js +++ b/js/xpconnect/tests/unit/head_watchdog.js @@ -10,6 +10,17 @@ const Cc = Components.classes; const Ci = Components.interfaces; const Cu = Components.utils; +Cu.import("resource://testing-common/PromiseTestUtils.jsm"); + +/////////////////// +// +// Whitelisting these tests. +// As part of bug 1077403, the shutdown crash should be fixed. +// +// These tests may crash intermittently on shutdown if the DOM Promise uncaught +// rejection observers are still registered when the watchdog operates. +PromiseTestUtils.thisTestLeaksUncaughtRejectionsAndShouldBeFixed(); + var gPrefs = Cc["@mozilla.org/preferences-service;1"].getService(Ci.nsIPrefBranch); function setWatchdogEnabled(enabled) { diff --git a/services/docs/index.rst b/services/docs/index.rst deleted file mode 100644 index a6e71ab83e..0000000000 --- a/services/docs/index.rst +++ /dev/null @@ -1,16 +0,0 @@ -======================= -Firefox Services Module -======================= - -The ``/services`` directory contains code for a variety of application -features that communicate with external services - hence its name. - -It was originally created to hold code for Firefox Sync. Later, it -became the location for code written by the Mozilla Services Client team -and thus includes :ref:`healthreport`. This team no longer exists, but -the directory remains. - -.. toctree:: - :maxdepth: 1 - - metrics diff --git a/services/docs/metrics.rst b/services/docs/metrics.rst deleted file mode 100644 index 41fc3ea2d4..0000000000 --- a/services/docs/metrics.rst +++ /dev/null @@ -1,130 +0,0 @@ -.. _services_metrics: - -============================ -Metrics Collection Framework -============================ - -The ``services/metrics`` directory contains a generic data metrics -collecting and persisting framework for Gecko applications. - -Overview -======== - -The Metrics framework by itself doesn't do much: it simply provides a -generic mechanism for collecting and persisting data. It is up to users -of this framework to drive collection and do something with the obtained -data. A consumer of this framework is :ref:`healthreport`. - -Relationship to Telemetry -------------------------- - -Telemetry provides similar features to code in this directory. The two -may be unified in the future. - -Usage -===== - -To use the code in this directory, import Metrics.jsm. e.g. - - Components.utils.import("resource://gre/modules/Metrics.jsm"); - -This exports a *Metrics* object which holds references to the main JS -types and functions provided by this feature. Read below for what those -types are. - -Metrics Types -============= - -``Metrics.jsm`` exports a number of types. They are documented in the -sections below. - -Metrics.Provider ----------------- - -``Metrics.Provider`` is an entity that collects and manages data. Providers -are typically domain-specific: if you need to collect a new type of data, -you create a ``Metrics.Provider`` type that does this. - -Metrics.Measurement -------------------- - -A ``Metrics.Measurement`` represents a collection of related pieces/fields -of data. - -All data recorded by the metrics framework is modeled as -``Metrics.Measurement`` instances. Instances of ``Metrics.Measurement`` -are essentially data structure descriptors. - -Each ``Metrics.Measurement`` consists of a name and version to identify -itself (and its data) as well as a list of *fields* that this measurement -holds. A *field* is effectively an entry in a data structure. It consists -of a name and strongly enumerated type. - -Metrics.Storage ---------------- - -This entity is responsible for persisting collected data and state. - -It currently uses SQLite to store data, but this detail is abstracted away -in order to facilitate swapping of storage backends. - -Metrics.ProviderManager ------------------------ - -High-level entity coordinating activity among several ``Metrics.Provider`` -instances. - -Providers and Measurements -========================== - -The most important types in this framework are ``Metrics.Provider`` and -``Metrics.Measurement``, henceforth known as ``Provider`` and -``Measurement``, respectively. As you will see, these two types go -hand in hand. - -A ``Provider`` is an entity that *provides* data about a specific subsystem -or feature. They do this by recording data to specific ``Measurement`` -types. Both ``Provider`` and ``Measurement`` are abstract base types. - -A ``Measurement`` implementation defines a name and version. More -importantly, it also defines its storage requirements and how -previously-stored values are serialized. - -Storage allocation is performed by communicating with the SQLite -backend. There is a startup function that tells SQLite what fields the -measurement is recording. The storage backend then registers these in -the database. Internally, this is creating a new primary key for -individual fields so later storage operations can directly reference -these primary keys in order to retrieve data without having to perform -complicated joins. - -A ``Provider`` can be thought of as a collection of ``Measurement`` -implementations. e.g. an Addons provider may consist of a measurement -for all *current* add-ons as well as a separate measurement for -historical counts of add-ons. A provider's primary role is to take -metrics data and write it to various measurements. This effectively -persists the data to SQLite. - -Data is emitted from providers in either a push or pull based mechanism. -In push-based scenarios, the provider likely subscribes to external -events (e.g. observer notifications). An event of interest can occur at -any time. When it does, the provider immediately writes the event of -interest to storage or buffers it for eventual writing. In pull-based -scenarios, the provider is periodically queried and asked to populate -data. - -SQLite Storage -============== - -``Metrics.Storage`` provides an interface for persisting metrics data to a -SQLite database. - -The storage API organizes values by fields. A field is a named member of -a ``Measurement`` that has specific type and retention characteristics. -Some example field types include: - -* Last text value -* Last numeric value for a given day -* Discrete text values for a given day - -See ``storage.jsm`` for more. diff --git a/services/metrics/Metrics.jsm b/services/metrics/Metrics.jsm deleted file mode 100644 index e1049d4687..0000000000 --- a/services/metrics/Metrics.jsm +++ /dev/null @@ -1,38 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MERGED_COMPARTMENT - -"use strict"; - -this.EXPORTED_SYMBOLS = ["Metrics"]; - -const {classes: Cc, interfaces: Ci, utils: Cu} = Components; - -const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000; - -#endif - -// We concatenate the JSMs together to eliminate compartment overhead. -// This is a giant hack until compartment overhead is no longer an -// issue. -#define MERGED_COMPARTMENT - -#include providermanager.jsm -; -#include dataprovider.jsm -; -#include storage.jsm -; - -this.Metrics = { - ProviderManager: ProviderManager, - DailyValues: DailyValues, - Measurement: Measurement, - Provider: Provider, - Storage: MetricsStorageBackend, - dateToDays: dateToDays, - daysToDate: daysToDate, -}; - diff --git a/services/metrics/dataprovider.jsm b/services/metrics/dataprovider.jsm deleted file mode 100644 index 0cb5d7fcdf..0000000000 --- a/services/metrics/dataprovider.jsm +++ /dev/null @@ -1,727 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MERGED_COMPARTMENT - -"use strict"; - -this.EXPORTED_SYMBOLS = [ - "Measurement", - "Provider", -]; - -const {utils: Cu} = Components; - -const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000; - -#endif - -Cu.import("resource://gre/modules/Promise.jsm"); -Cu.import("resource://gre/modules/Preferences.jsm"); -Cu.import("resource://gre/modules/Task.jsm"); -Cu.import("resource://gre/modules/Log.jsm"); -Cu.import("resource://services-common/utils.js"); - - - -/** - * Represents a collection of related pieces/fields of data. - * - * This is an abstract base type. - * - * This type provides the primary interface for storing, retrieving, and - * serializing data. - * - * Each measurement consists of a set of named fields. Each field is primarily - * identified by a string name, which must be unique within the measurement. - * - * Each derived type must define the following properties: - * - * name -- String name of this measurement. This is the primary way - * measurements are distinguished within a provider. - * - * version -- Integer version of this measurement. This is a secondary - * identifier for a measurement within a provider. The version denotes - * the behavior of this measurement and the composition of its fields over - * time. When a new field is added or the behavior of an existing field - * changes, the version should be incremented. The initial version of a - * measurement is typically 1. - * - * fields -- Object defining the fields this measurement holds. Keys in the - * object are string field names. Values are objects describing how the - * field works. The following properties are recognized: - * - * type -- The string type of this field. This is typically one of the - * FIELD_* constants from the Metrics.Storage type. - * - * - * FUTURE: provide hook points for measurements to supplement with custom - * storage needs. - */ -this.Measurement = function () { - if (!this.name) { - throw new Error("Measurement must have a name."); - } - - if (!this.version) { - throw new Error("Measurement must have a version."); - } - - if (!Number.isInteger(this.version)) { - throw new Error("Measurement's version must be an integer: " + this.version); - } - - if (!this.fields) { - throw new Error("Measurement must define fields."); - } - - for (let [name, info] in Iterator(this.fields)) { - if (!info) { - throw new Error("Field does not contain metadata: " + name); - } - - if (!info.type) { - throw new Error("Field is missing required type property: " + name); - } - } - - this._log = Log.repository.getLogger("Services.Metrics.Measurement." + this.name); - - this.id = null; - this.storage = null; - this._fields = {}; - - this._serializers = {}; - this._serializers[this.SERIALIZE_JSON] = { - singular: this._serializeJSONSingular.bind(this), - daily: this._serializeJSONDay.bind(this), - }; -} - -Measurement.prototype = Object.freeze({ - SERIALIZE_JSON: "json", - - /** - * Obtain a serializer for this measurement. - * - * Implementations should return an object with the following keys: - * - * singular -- Serializer for singular data. - * daily -- Serializer for daily data. - * - * Each item is a function that takes a single argument: the data to - * serialize. The passed data is a subset of that returned from - * this.getValues(). For "singular," data.singular is passed. For "daily", - * data.days.get() is passed. - * - * This function receives a single argument: the serialization format we - * are requesting. This is one of the SERIALIZE_* constants on this base type. - * - * For SERIALIZE_JSON, the function should return an object that - * JSON.stringify() knows how to handle. This could be an anonymous object or - * array or any object with a property named `toJSON` whose value is a - * function. The returned object will be added to a larger document - * containing the results of all `serialize` calls. - * - * The default implementation knows how to serialize built-in types using - * very simple logic. If small encoding size is a goal, the default - * implementation may not be suitable. If an unknown field type is - * encountered, the default implementation will error. - * - * @param format - * (string) A SERIALIZE_* constant defining what serialization format - * to use. - */ - serializer: function (format) { - if (!(format in this._serializers)) { - throw new Error("Don't know how to serialize format: " + format); - } - - return this._serializers[format]; - }, - - /** - * Whether this measurement contains the named field. - * - * @param name - * (string) Name of field. - * - * @return bool - */ - hasField: function (name) { - return name in this.fields; - }, - - /** - * The unique identifier for a named field. - * - * This will throw if the field is not known. - * - * @param name - * (string) Name of field. - */ - fieldID: function (name) { - let entry = this._fields[name]; - - if (!entry) { - throw new Error("Unknown field: " + name); - } - - return entry[0]; - }, - - fieldType: function (name) { - let entry = this._fields[name]; - - if (!entry) { - throw new Error("Unknown field: " + name); - } - - return entry[1]; - }, - - _configureStorage: function () { - let missing = []; - for (let [name, info] in Iterator(this.fields)) { - if (this.storage.hasFieldFromMeasurement(this.id, name)) { - this._fields[name] = - [this.storage.fieldIDFromMeasurement(this.id, name), info.type]; - continue; - } - - missing.push([name, info.type]); - } - - if (!missing.length) { - return CommonUtils.laterTickResolvingPromise(); - } - - // We only perform a transaction if we have work to do (to avoid - // extra SQLite overhead). - return this.storage.enqueueTransaction(function registerFields() { - for (let [name, type] of missing) { - this._log.debug("Registering field: " + name + " " + type); - let id = yield this.storage.registerField(this.id, name, type); - this._fields[name] = [id, type]; - } - }.bind(this)); - }, - - //--------------------------------------------------------------------------- - // Data Recording Functions - // - // Functions in this section are used to record new values against this - // measurement instance. - // - // Generally speaking, these functions will throw if the specified field does - // not exist or if the storage function requested is not appropriate for the - // type of that field. These functions will also return a promise that will - // be resolved when the underlying storage operation has completed. - //--------------------------------------------------------------------------- - - /** - * Increment a daily counter field in this measurement by 1. - * - * By default, the counter for the current day will be incremented. - * - * If the field is not known or is not a daily counter, this will throw. - * - * - * - * @param field - * (string) The name of the field whose value to increment. - * @param date - * (Date) Day on which to increment the counter. - * @param by - * (integer) How much to increment by. - * @return Promise<> - */ - incrementDailyCounter: function (field, date=new Date(), by=1) { - return this.storage.incrementDailyCounterFromFieldID(this.fieldID(field), - date, by); - }, - - /** - * Record a new numeric value for a daily discrete numeric field. - * - * @param field - * (string) The name of the field to append a value to. - * @param value - * (Number) Number to append. - * @param date - * (Date) Day on which to append the value. - * - * @return Promise<> - */ - addDailyDiscreteNumeric: function (field, value, date=new Date()) { - return this.storage.addDailyDiscreteNumericFromFieldID( - this.fieldID(field), value, date); - }, - - /** - * Record a new text value for a daily discrete text field. - * - * This is like `addDailyDiscreteNumeric` but for daily discrete text fields. - */ - addDailyDiscreteText: function (field, value, date=new Date()) { - return this.storage.addDailyDiscreteTextFromFieldID( - this.fieldID(field), value, date); - }, - - /** - * Record the last seen value for a last numeric field. - * - * @param field - * (string) The name of the field to set the value of. - * @param value - * (Number) The value to set. - * @param date - * (Date) When this value was recorded. - * - * @return Promise<> - */ - setLastNumeric: function (field, value, date=new Date()) { - return this.storage.setLastNumericFromFieldID(this.fieldID(field), value, - date); - }, - - /** - * Record the last seen value for a last text field. - * - * This is like `setLastNumeric` except for last text fields. - */ - setLastText: function (field, value, date=new Date()) { - return this.storage.setLastTextFromFieldID(this.fieldID(field), value, - date); - }, - - /** - * Record the most recent value for a daily last numeric field. - * - * @param field - * (string) The name of a daily last numeric field. - * @param value - * (Number) The value to set. - * @param date - * (Date) Day on which to record the last value. - * - * @return Promise<> - */ - setDailyLastNumeric: function (field, value, date=new Date()) { - return this.storage.setDailyLastNumericFromFieldID(this.fieldID(field), - value, date); - }, - - /** - * Record the most recent value for a daily last text field. - * - * This is like `setDailyLastNumeric` except for a daily last text field. - */ - setDailyLastText: function (field, value, date=new Date()) { - return this.storage.setDailyLastTextFromFieldID(this.fieldID(field), - value, date); - }, - - //--------------------------------------------------------------------------- - // End of data recording APIs. - //--------------------------------------------------------------------------- - - /** - * Obtain all values stored for this measurement. - * - * The default implementation obtains all known types from storage. If the - * measurement provides custom types or stores values somewhere other than - * storage, it should define its own implementation. - * - * This returns a promise that resolves to a data structure which is - * understood by the measurement's serialize() function. - */ - getValues: function () { - return this.storage.getMeasurementValues(this.id); - }, - - deleteLastNumeric: function (field) { - return this.storage.deleteLastNumericFromFieldID(this.fieldID(field)); - }, - - deleteLastText: function (field) { - return this.storage.deleteLastTextFromFieldID(this.fieldID(field)); - }, - - /** - * This method is used by the default serializers to control whether a field - * is included in the output. - * - * There could be legacy fields in storage we no longer care about. - * - * This method is a hook to allow measurements to change this behavior, e.g., - * to implement a dynamic fieldset. - * - * You will also need to override `fieldType`. - * - * @return (boolean) true if the specified field should be included in - * payload output. - */ - shouldIncludeField: function (field) { - return field in this._fields; - }, - - _serializeJSONSingular: function (data) { - let result = {"_v": this.version}; - - for (let [field, value] of data) { - // There could be legacy fields in storage we no longer care about. - if (!this.shouldIncludeField(field)) { - continue; - } - - let type = this.fieldType(field); - - switch (type) { - case this.storage.FIELD_LAST_NUMERIC: - case this.storage.FIELD_LAST_TEXT: - result[field] = value[1]; - break; - - case this.storage.FIELD_DAILY_COUNTER: - case this.storage.FIELD_DAILY_DISCRETE_NUMERIC: - case this.storage.FIELD_DAILY_DISCRETE_TEXT: - case this.storage.FIELD_DAILY_LAST_NUMERIC: - case this.storage.FIELD_DAILY_LAST_TEXT: - continue; - - default: - throw new Error("Unknown field type: " + type); - } - } - - return result; - }, - - _serializeJSONDay: function (data) { - let result = {"_v": this.version}; - - for (let [field, value] of data) { - if (!this.shouldIncludeField(field)) { - continue; - } - - let type = this.fieldType(field); - - switch (type) { - case this.storage.FIELD_DAILY_COUNTER: - case this.storage.FIELD_DAILY_DISCRETE_NUMERIC: - case this.storage.FIELD_DAILY_DISCRETE_TEXT: - case this.storage.FIELD_DAILY_LAST_NUMERIC: - case this.storage.FIELD_DAILY_LAST_TEXT: - result[field] = value; - break; - - case this.storage.FIELD_LAST_NUMERIC: - case this.storage.FIELD_LAST_TEXT: - continue; - - default: - throw new Error("Unknown field type: " + type); - } - } - - return result; - }, -}); - - -/** - * An entity that emits data. - * - * A `Provider` consists of a string name (must be globally unique among all - * known providers) and a set of `Measurement` instances. - * - * The main role of a `Provider` is to produce metrics data and to store said - * data in the storage backend. - * - * Metrics data collection is initiated either by a manager calling a - * `collect*` function on `Provider` instances or by the `Provider` registering - * to some external event and then reacting whenever they occur. - * - * `Provider` implementations interface directly with a storage backend. For - * common stored values (daily counters, daily discrete values, etc), - * implementations should interface with storage via the various helper - * functions on the `Measurement` instances. For custom stored value types, - * implementations will interact directly with the low-level storage APIs. - * - * Because multiple providers exist and could be responding to separate - * external events simultaneously and because not all operations performed by - * storage can safely be performed in parallel, writing directly to storage at - * event time is dangerous. Therefore, interactions with storage must be - * deferred until it is safe to perform them. - * - * This typically looks something like: - * - * // This gets called when an external event worthy of recording metrics - * // occurs. The function receives a numeric value associated with the event. - * function onExternalEvent (value) { - * let now = new Date(); - * let m = this.getMeasurement("foo", 1); - * - * this.enqueueStorageOperation(function storeExternalEvent() { - * - * // We interface with storage via the `Measurement` helper functions. - * // These each return a promise that will be resolved when the - * // operation finishes. We rely on behavior of storage where operations - * // are executed single threaded and sequentially. Therefore, we only - * // need to return the final promise. - * m.incrementDailyCounter("foo", now); - * return m.addDailyDiscreteNumericValue("my_value", value, now); - * }.bind(this)); - * - * } - * - * - * `Provider` is an abstract base class. Implementations must define a few - * properties: - * - * name - * The `name` property should be a string defining the provider's name. The - * name must be globally unique for the application. The name is used as an - * identifier to distinguish providers from each other. - * - * measurementTypes - * This must be an array of `Measurement`-derived types. Note that elements - * in the array are the type functions, not instances. Instances of the - * `Measurement` are created at run-time by the `Provider` and are bound - * to the provider and to a specific storage backend. - */ -this.Provider = function () { - if (!this.name) { - throw new Error("Provider must define a name."); - } - - if (!Array.isArray(this.measurementTypes)) { - throw new Error("Provider must define measurement types."); - } - - this._log = Log.repository.getLogger("Services.Metrics.Provider." + this.name); - - this.measurements = null; - this.storage = null; -} - -Provider.prototype = Object.freeze({ - /** - * Whether the provider only pulls data from other sources. - * - * If this is true, the provider pulls data from other sources. By contrast, - * "push-based" providers subscribe to foreign sources and record/react to - * external events as they happen. - * - * Pull-only providers likely aren't instantiated until a data collection - * is performed. Thus, implementations cannot rely on a provider instance - * always being alive. This is an optimization so provider instances aren't - * dead weight while the application is running. - * - * This must be set on the prototype to have an effect. - */ - pullOnly: false, - - /** - * Obtain a `Measurement` from its name and version. - * - * If the measurement is not found, an Error is thrown. - */ - getMeasurement: function (name, version) { - if (!Number.isInteger(version)) { - throw new Error("getMeasurement expects an integer version. Got: " + version); - } - - let m = this.measurements.get([name, version].join(":")); - - if (!m) { - throw new Error("Unknown measurement: " + name + " v" + version); - } - - return m; - }, - - init: function (storage) { - if (this.storage !== null) { - throw new Error("Provider() not called. Did the sub-type forget to call it?"); - } - - if (this.storage) { - throw new Error("Provider has already been initialized."); - } - - this.measurements = new Map(); - this.storage = storage; - - let self = this; - return Task.spawn(function init() { - let pre = self.preInit(); - if (!pre || typeof(pre.then) != "function") { - throw new Error("preInit() does not return a promise."); - } - yield pre; - - for (let measurementType of self.measurementTypes) { - let measurement = new measurementType(); - - measurement.provider = self; - measurement.storage = self.storage; - - let id = yield storage.registerMeasurement(self.name, measurement.name, - measurement.version); - - measurement.id = id; - - yield measurement._configureStorage(); - - self.measurements.set([measurement.name, measurement.version].join(":"), - measurement); - } - - let post = self.postInit(); - if (!post || typeof(post.then) != "function") { - throw new Error("postInit() does not return a promise."); - } - yield post; - }); - }, - - shutdown: function () { - let promise = this.onShutdown(); - - if (!promise || typeof(promise.then) != "function") { - throw new Error("onShutdown implementation does not return a promise."); - } - - return promise; - }, - - /** - * Hook point for implementations to perform pre-initialization activity. - * - * This method will be called before measurement registration. - * - * Implementations should return a promise which is resolved when - * initialization activities have completed. - */ - preInit: function () { - return CommonUtils.laterTickResolvingPromise(); - }, - - /** - * Hook point for implementations to perform post-initialization activity. - * - * This method will be called after `preInit` and measurement registration, - * but before initialization is finished. - * - * If a `Provider` instance needs to register observers, etc, it should - * implement this function. - * - * Implementations should return a promise which is resolved when - * initialization activities have completed. - */ - postInit: function () { - return CommonUtils.laterTickResolvingPromise(); - }, - - /** - * Hook point for shutdown of instances. - * - * This is the opposite of `onInit`. If a `Provider` needs to unregister - * observers, etc, this is where it should do it. - * - * Implementations should return a promise which is resolved when - * shutdown activities have completed. - */ - onShutdown: function () { - return CommonUtils.laterTickResolvingPromise(); - }, - - /** - * Collects data that doesn't change during the application's lifetime. - * - * Implementations should return a promise that resolves when all data has - * been collected and storage operations have been finished. - * - * @return Promise<> - */ - collectConstantData: function () { - return CommonUtils.laterTickResolvingPromise(); - }, - - /** - * Collects data approximately every day. - * - * For long-running applications, this is called approximately every day. - * It may or may not be called every time the application is run. It also may - * be called more than once per day. - * - * Implementations should return a promise that resolves when all data has - * been collected and storage operations have completed. - * - * @return Promise<> - */ - collectDailyData: function () { - return CommonUtils.laterTickResolvingPromise(); - }, - - /** - * Queue a deferred storage operation. - * - * Deferred storage operations are the preferred method for providers to - * interact with storage. When collected data is to be added to storage, - * the provider creates a function that performs the necessary storage - * interactions and then passes that function to this function. Pending - * storage operations will be executed sequentially by a coordinator. - * - * The passed function should return a promise which will be resolved upon - * completion of storage interaction. - */ - enqueueStorageOperation: function (func) { - return this.storage.enqueueOperation(func); - }, - - /** - * Obtain persisted provider state. - * - * Provider state consists of key-value pairs of string names and values. - * Providers can stuff whatever they want into state. They are encouraged to - * store as little as possible for performance reasons. - * - * State is backed by storage and is robust. - * - * These functions do not enqueue on storage automatically, so they should - * be guarded by `enqueueStorageOperation` or some other mutex. - * - * @param key - * (string) The property to retrieve. - * - * @return Promise String value on success. null if no state - * is available under this key. - */ - getState: function (key) { - return this.storage.getProviderState(this.name, key); - }, - - /** - * Set state for this provider. - * - * This is the complementary API for `getState` and obeys the same - * storage restrictions. - */ - setState: function (key, value) { - return this.storage.setProviderState(this.name, key, value); - }, - - _dateToDays: function (date) { - return Math.floor(date.getTime() / MILLISECONDS_PER_DAY); - }, - - _daysToDate: function (days) { - return new Date(days * MILLISECONDS_PER_DAY); - }, -}); - diff --git a/services/metrics/modules-testing/mocks.jsm b/services/metrics/modules-testing/mocks.jsm deleted file mode 100644 index 032f167a96..0000000000 --- a/services/metrics/modules-testing/mocks.jsm +++ /dev/null @@ -1,154 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -"use strict"; - -this.EXPORTED_SYMBOLS = [ - "DummyMeasurement", - "DummyProvider", - "DummyConstantProvider", - "DummyPullOnlyThrowsOnInitProvider", - "DummyThrowOnInitProvider", - "DummyThrowOnShutdownProvider", -]; - -const {utils: Cu} = Components; - -Cu.import("resource://gre/modules/Promise.jsm"); -Cu.import("resource://gre/modules/Metrics.jsm"); -Cu.import("resource://gre/modules/Task.jsm"); - -this.DummyMeasurement = function DummyMeasurement(name="DummyMeasurement") { - this.name = name; - - Metrics.Measurement.call(this); -} - -DummyMeasurement.prototype = { - __proto__: Metrics.Measurement.prototype, - - version: 1, - - fields: { - "daily-counter": {type: Metrics.Storage.FIELD_DAILY_COUNTER}, - "daily-discrete-numeric": {type: Metrics.Storage.FIELD_DAILY_DISCRETE_NUMERIC}, - "daily-discrete-text": {type: Metrics.Storage.FIELD_DAILY_DISCRETE_TEXT}, - "daily-last-numeric": {type: Metrics.Storage.FIELD_DAILY_LAST_NUMERIC}, - "daily-last-text": {type: Metrics.Storage.FIELD_DAILY_LAST_TEXT}, - "last-numeric": {type: Metrics.Storage.FIELD_LAST_NUMERIC}, - "last-text": {type: Metrics.Storage.FIELD_LAST_TEXT}, - }, -}; - - -this.DummyProvider = function DummyProvider(name="DummyProvider") { - Object.defineProperty(this, "name", { - value: name, - }); - - this.measurementTypes = [DummyMeasurement]; - - Metrics.Provider.call(this); - - this.constantMeasurementName = "DummyMeasurement"; - this.collectConstantCount = 0; - this.throwDuringCollectConstantData = null; - this.throwDuringConstantPopulate = null; - - this.collectDailyCount = 0; - - this.havePushedMeasurements = true; -} - -DummyProvider.prototype = { - __proto__: Metrics.Provider.prototype, - - name: "DummyProvider", - - collectConstantData: function () { - this.collectConstantCount++; - - if (this.throwDuringCollectConstantData) { - throw new Error(this.throwDuringCollectConstantData); - } - - return this.enqueueStorageOperation(function doStorage() { - if (this.throwDuringConstantPopulate) { - throw new Error(this.throwDuringConstantPopulate); - } - - let m = this.getMeasurement("DummyMeasurement", 1); - let now = new Date(); - m.incrementDailyCounter("daily-counter", now); - m.addDailyDiscreteNumeric("daily-discrete-numeric", 1, now); - m.addDailyDiscreteNumeric("daily-discrete-numeric", 2, now); - m.addDailyDiscreteText("daily-discrete-text", "foo", now); - m.addDailyDiscreteText("daily-discrete-text", "bar", now); - m.setDailyLastNumeric("daily-last-numeric", 3, now); - m.setDailyLastText("daily-last-text", "biz", now); - m.setLastNumeric("last-numeric", 4, now); - return m.setLastText("last-text", "bazfoo", now); - }.bind(this)); - }, - - collectDailyData: function () { - this.collectDailyCount++; - - return Promise.resolve(); - }, -}; - - -this.DummyConstantProvider = function () { - DummyProvider.call(this, this.name); -} - -DummyConstantProvider.prototype = { - __proto__: DummyProvider.prototype, - - name: "DummyConstantProvider", - - pullOnly: true, -}; - -this.DummyThrowOnInitProvider = function () { - DummyProvider.call(this, "DummyThrowOnInitProvider"); - - throw new Error("Dummy Error"); -}; - -this.DummyThrowOnInitProvider.prototype = { - __proto__: DummyProvider.prototype, - - name: "DummyThrowOnInitProvider", -}; - -this.DummyPullOnlyThrowsOnInitProvider = function () { - DummyConstantProvider.call(this); - - throw new Error("Dummy Error"); -}; - -this.DummyPullOnlyThrowsOnInitProvider.prototype = { - __proto__: DummyConstantProvider.prototype, - - name: "DummyPullOnlyThrowsOnInitProvider", -}; - -this.DummyThrowOnShutdownProvider = function () { - DummyProvider.call(this, "DummyThrowOnShutdownProvider"); -}; - -this.DummyThrowOnShutdownProvider.prototype = { - __proto__: DummyProvider.prototype, - - name: "DummyThrowOnShutdownProvider", - - pullOnly: true, - - onShutdown: function () { - throw new Error("Dummy shutdown error"); - }, -}; - diff --git a/services/metrics/moz.build b/services/metrics/moz.build deleted file mode 100644 index a7ce296a95..0000000000 --- a/services/metrics/moz.build +++ /dev/null @@ -1,23 +0,0 @@ -# -*- Mode: python; c-basic-offset: 4; indent-tabs-mode: nil; tab-width: 40 -*- -# vim: set filetype=python: -# This Source Code Form is subject to the terms of the Mozilla Public -# License, v. 2.0. If a copy of the MPL was not distributed with this -# file, You can obtain one at http://mozilla.org/MPL/2.0/. - -XPCSHELL_TESTS_MANIFESTS += ['tests/xpcshell/xpcshell.ini'] - -# We install Metrics.jsm into the "main" JSM repository and the rest in -# services. External consumers should only go through Metrics.jsm. -EXTRA_PP_JS_MODULES += [ - 'Metrics.jsm', -] - -EXTRA_PP_JS_MODULES.services.metrics += [ - 'dataprovider.jsm', - 'providermanager.jsm', - 'storage.jsm', -] - -TESTING_JS_MODULES.services.metrics += [ - 'modules-testing/mocks.jsm', -] diff --git a/services/metrics/providermanager.jsm b/services/metrics/providermanager.jsm deleted file mode 100644 index cf26d444b2..0000000000 --- a/services/metrics/providermanager.jsm +++ /dev/null @@ -1,562 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MERGED_COMPARTMENT -"use strict"; - -this.EXPORTED_SYMBOLS = ["ProviderManager"]; - -const {classes: Cc, interfaces: Ci, utils: Cu} = Components; - -Cu.import("resource://gre/modules/services/metrics/dataprovider.jsm"); -#endif - -Cu.import("resource://gre/modules/Promise.jsm"); -Cu.import("resource://gre/modules/Task.jsm"); -Cu.import("resource://gre/modules/Log.jsm"); -Cu.import("resource://services-common/utils.js"); - - -/** - * Handles and coordinates the collection of metrics data from providers. - * - * This provides an interface for managing `Metrics.Provider` instances. It - * provides APIs for bulk collection of data. - */ -this.ProviderManager = function (storage) { - this._log = Log.repository.getLogger("Services.Metrics.ProviderManager"); - - this._providers = new Map(); - this._storage = storage; - - this._providerInitQueue = []; - this._providerInitializing = false; - - this._pullOnlyProviders = {}; - this._pullOnlyProvidersRegisterCount = 0; - this._pullOnlyProvidersState = this.PULL_ONLY_NOT_REGISTERED; - this._pullOnlyProvidersCurrentPromise = null; - - // Callback to allow customization of providers after they are constructed - // but before they call out into their initialization code. - this.onProviderInit = null; -} - -this.ProviderManager.prototype = Object.freeze({ - PULL_ONLY_NOT_REGISTERED: "none", - PULL_ONLY_REGISTERING: "registering", - PULL_ONLY_UNREGISTERING: "unregistering", - PULL_ONLY_REGISTERED: "registered", - - get providers() { - let providers = []; - for (let [name, entry] of this._providers) { - providers.push(entry.provider); - } - - return providers; - }, - - /** - * Obtain a provider from its name. - */ - getProvider: function (name) { - let provider = this._providers.get(name); - - if (!provider) { - return null; - } - - return provider.provider; - }, - - /** - * Registers providers from a category manager category. - * - * This examines the specified category entries and registers found - * providers. - * - * Category entries are essentially JS modules and the name of the symbol - * within that module that is a `Metrics.Provider` instance. - * - * The category entry name is the name of the JS type for the provider. The - * value is the resource:// URI to import which makes this type available. - * - * Example entry: - * - * FooProvider resource://gre/modules/foo.jsm - * - * One can register entries in the application's .manifest file. e.g. - * - * category healthreport-js-provider-default FooProvider resource://gre/modules/foo.jsm - * category healthreport-js-provider-nightly EyeballProvider resource://gre/modules/eyeball.jsm - * - * Then to load them: - * - * let reporter = getHealthReporter("healthreport."); - * reporter.registerProvidersFromCategoryManager("healthreport-js-provider-default"); - * - * If the category has no defined members, this call has no effect, and no error is raised. - * - * @param category - * (string) Name of category from which to query and load. - * @param providerDiagnostic - * (function) Optional, called with the name of the provider currently being initialized. - * @return a newly spawned Task. - */ - registerProvidersFromCategoryManager: function (category, providerDiagnostic) { - this._log.info("Registering providers from category: " + category); - let cm = Cc["@mozilla.org/categorymanager;1"] - .getService(Ci.nsICategoryManager); - - let promiseList = []; - let enumerator = cm.enumerateCategory(category); - while (enumerator.hasMoreElements()) { - let entry = enumerator.getNext() - .QueryInterface(Ci.nsISupportsCString) - .toString(); - - let uri = cm.getCategoryEntry(category, entry); - this._log.info("Attempting to load provider from category manager: " + - entry + " from " + uri); - - try { - let ns = {}; - Cu.import(uri, ns); - - let promise = this.registerProviderFromType(ns[entry]); - if (promise) { - promiseList.push({name: entry, promise: promise}); - } - } catch (ex) { - this._recordProviderError(entry, - "Error registering provider from category manager", - ex); - continue; - } - } - - return Task.spawn(function* wait() { - for (let entry of promiseList) { - if (providerDiagnostic) { - providerDiagnostic(entry.name); - } - yield entry.promise; - } - }); - }, - - /** - * Registers a `MetricsProvider` with this manager. - * - * Once a `MetricsProvider` is registered, data will be collected from it - * whenever we collect data. - * - * The returned value is a promise that will be resolved once registration - * is complete. - * - * Providers are initialized as part of registration by calling - * provider.init(). - * - * @param provider - * (Metrics.Provider) The provider instance to register. - * - * @return Promise - */ - registerProvider: function (provider) { - // We should perform an instanceof check here. However, due to merged - // compartments, the Provider type may belong to one of two JSMs - // isinstance gets confused depending on which module Provider comes - // from. Some code references Provider from dataprovider.jsm; others from - // Metrics.jsm. - if (!provider.name) { - throw new Error("Provider is not valid: does not have a name."); - } - if (this._providers.has(provider.name)) { - return CommonUtils.laterTickResolvingPromise(); - } - - let deferred = Promise.defer(); - this._providerInitQueue.push([provider, deferred]); - - if (this._providerInitQueue.length == 1) { - this._popAndInitProvider(); - } - - return deferred.promise; - }, - - /** - * Registers a provider from its constructor function. - * - * If the provider is pull-only, it will be stashed away and - * initialized later. Null will be returned. - * - * If it is not pull-only, it will be initialized immediately and a - * promise will be returned. The promise will be resolved when the - * provider has finished initializing. - */ - registerProviderFromType: function (type) { - let proto = type.prototype; - if (proto.pullOnly) { - this._log.info("Provider is pull-only. Deferring initialization: " + - proto.name); - this._pullOnlyProviders[proto.name] = type; - - return null; - } - - let provider = this._initProviderFromType(type); - return this.registerProvider(provider); - }, - - /** - * Initializes a provider from its type. - * - * This is how a constructor function should be turned into a provider - * instance. - * - * A side-effect is the provider is registered with the manager. - */ - _initProviderFromType: function (type) { - let provider = new type(); - if (this.onProviderInit) { - this.onProviderInit(provider); - } - - return provider; - }, - - /** - * Remove a named provider from the manager. - * - * It is the caller's responsibility to shut down the provider - * instance. - */ - unregisterProvider: function (name) { - this._providers.delete(name); - }, - - /** - * Ensure that pull-only providers are registered. - */ - ensurePullOnlyProvidersRegistered: function () { - let state = this._pullOnlyProvidersState; - - this._pullOnlyProvidersRegisterCount++; - - if (state == this.PULL_ONLY_REGISTERED) { - this._log.debug("Requested pull-only provider registration and " + - "providers are already registered."); - return CommonUtils.laterTickResolvingPromise(); - } - - // If we're in the process of registering, chain off that request. - if (state == this.PULL_ONLY_REGISTERING) { - this._log.debug("Requested pull-only provider registration and " + - "registration is already in progress."); - return this._pullOnlyProvidersCurrentPromise; - } - - this._log.debug("Pull-only provider registration requested."); - - // A side-effect of setting this is that an active unregistration will - // effectively short circuit and finish as soon as the in-flight - // unregistration (if any) finishes. - this._pullOnlyProvidersState = this.PULL_ONLY_REGISTERING; - - let inFlightPromise = this._pullOnlyProvidersCurrentPromise; - - this._pullOnlyProvidersCurrentPromise = - Task.spawn(function registerPullProviders() { - - if (inFlightPromise) { - this._log.debug("Waiting for in-flight pull-only provider activity " + - "to finish before registering."); - try { - yield inFlightPromise; - } catch (ex) { - this._log.warn("Error when waiting for existing pull-only promise: " + - CommonUtils.exceptionStr(ex)); - } - } - - for (let name in this._pullOnlyProviders) { - let providerType = this._pullOnlyProviders[name]; - // Short-circuit if we're no longer registering. - if (this._pullOnlyProvidersState != this.PULL_ONLY_REGISTERING) { - this._log.debug("Aborting pull-only provider registration."); - break; - } - - try { - let provider = this._initProviderFromType(providerType); - - // This is a no-op if the provider is already registered. So, the - // only overhead is constructing an instance. This should be cheap - // and isn't worth optimizing. - yield this.registerProvider(provider); - } catch (ex) { - this._recordProviderError(providerType.prototype.name, - "Error registering pull-only provider", - ex); - } - } - - // It's possible we changed state while registering. Only mark as - // registered if we didn't change state. - if (this._pullOnlyProvidersState == this.PULL_ONLY_REGISTERING) { - this._pullOnlyProvidersState = this.PULL_ONLY_REGISTERED; - this._pullOnlyProvidersCurrentPromise = null; - } - }.bind(this)); - return this._pullOnlyProvidersCurrentPromise; - }, - - ensurePullOnlyProvidersUnregistered: function () { - let state = this._pullOnlyProvidersState; - - // If we're not registered, this is a no-op. - if (state == this.PULL_ONLY_NOT_REGISTERED) { - this._log.debug("Requested pull-only provider unregistration but none " + - "are registered."); - return CommonUtils.laterTickResolvingPromise(); - } - - // If we're currently unregistering, recycle the promise from last time. - if (state == this.PULL_ONLY_UNREGISTERING) { - this._log.debug("Requested pull-only provider unregistration and " + - "unregistration is in progress."); - this._pullOnlyProvidersRegisterCount = - Math.max(0, this._pullOnlyProvidersRegisterCount - 1); - - return this._pullOnlyProvidersCurrentPromise; - } - - // We ignore this request while multiple entities have requested - // registration because we don't want a request from an "inner," - // short-lived request to overwrite the desire of the "parent," - // longer-lived request. - if (this._pullOnlyProvidersRegisterCount > 1) { - this._log.debug("Requested pull-only provider unregistration while " + - "other callers still want them registered. Ignoring."); - this._pullOnlyProvidersRegisterCount--; - return CommonUtils.laterTickResolvingPromise(); - } - - // We are either fully registered or registering with a single consumer. - // In both cases we are authoritative and can commence unregistration. - - this._log.debug("Pull-only providers being unregistered."); - this._pullOnlyProvidersRegisterCount = - Math.max(0, this._pullOnlyProvidersRegisterCount - 1); - this._pullOnlyProvidersState = this.PULL_ONLY_UNREGISTERING; - let inFlightPromise = this._pullOnlyProvidersCurrentPromise; - - this._pullOnlyProvidersCurrentPromise = - Task.spawn(function unregisterPullProviders() { - - if (inFlightPromise) { - this._log.debug("Waiting for in-flight pull-only provider activity " + - "to complete before unregistering."); - try { - yield inFlightPromise; - } catch (ex) { - this._log.warn("Error when waiting for existing pull-only promise: " + - CommonUtils.exceptionStr(ex)); - } - } - - for (let provider of this.providers) { - if (this._pullOnlyProvidersState != this.PULL_ONLY_UNREGISTERING) { - return; - } - - if (!provider.pullOnly) { - continue; - } - - this._log.info("Shutting down pull-only provider: " + - provider.name); - - try { - yield provider.shutdown(); - } catch (ex) { - this._recordProviderError(provider.name, - "Error when shutting down provider", - ex); - } finally { - this.unregisterProvider(provider.name); - } - } - - if (this._pullOnlyProvidersState == this.PULL_ONLY_UNREGISTERING) { - this._pullOnlyProvidersState = this.PULL_ONLY_NOT_REGISTERED; - this._pullOnlyProvidersCurrentPromise = null; - } - }.bind(this)); - return this._pullOnlyProvidersCurrentPromise; - }, - - _popAndInitProvider: function () { - if (!this._providerInitQueue.length || this._providerInitializing) { - return; - } - - let [provider, deferred] = this._providerInitQueue.shift(); - this._providerInitializing = true; - - this._log.info("Initializing provider with storage: " + provider.name); - - Task.spawn(function initProvider() { - try { - let result = yield provider.init(this._storage); - this._log.info("Provider successfully initialized: " + provider.name); - - this._providers.set(provider.name, { - provider: provider, - constantsCollected: false, - }); - - deferred.resolve(result); - } catch (ex) { - this._recordProviderError(provider.name, "Failed to initialize", ex); - deferred.reject(ex); - } finally { - this._providerInitializing = false; - this._popAndInitProvider(); - } - }.bind(this)); - }, - - /** - * Collects all constant measurements from all providers. - * - * Returns a Promise that will be fulfilled once all data providers have - * provided their constant data. A side-effect of this promise fulfillment - * is that the manager is populated with the obtained collection results. - * The resolved value to the promise is this `ProviderManager` instance. - * - * @param providerDiagnostic - * (function) Optional, called with the name of the provider currently being initialized. - */ - collectConstantData: function (providerDiagnostic=null) { - let entries = []; - - for (let [name, entry] of this._providers) { - if (entry.constantsCollected) { - this._log.trace("Provider has already provided constant data: " + - name); - continue; - } - - entries.push(entry); - } - - let onCollect = function (entry, result) { - entry.constantsCollected = true; - }; - - return this._callCollectOnProviders(entries, "collectConstantData", - onCollect, providerDiagnostic); - }, - - /** - * Calls collectDailyData on all providers. - */ - collectDailyData: function (providerDiagnostic=null) { - return this._callCollectOnProviders(this._providers.values(), - "collectDailyData", - null, - providerDiagnostic); - }, - - _callCollectOnProviders: function (entries, fnProperty, onCollect=null, providerDiagnostic=null) { - let promises = []; - - for (let entry of entries) { - let provider = entry.provider; - let collectPromise; - try { - collectPromise = provider[fnProperty].call(provider); - } catch (ex) { - this._recordProviderError(provider.name, "Exception when calling " + - "collect function: " + fnProperty, ex); - continue; - } - - if (!collectPromise) { - this._recordProviderError(provider.name, "Does not return a promise " + - "from " + fnProperty + "()"); - continue; - } - - let promise = collectPromise.then(function onCollected(result) { - if (onCollect) { - try { - onCollect(entry, result); - } catch (ex) { - this._log.warn("onCollect callback threw: " + - CommonUtils.exceptionStr(ex)); - } - } - - return CommonUtils.laterTickResolvingPromise(result); - }); - - promises.push([provider.name, promise]); - } - - return this._handleCollectionPromises(promises, providerDiagnostic); - }, - - /** - * Handles promises returned by the collect* functions. - * - * This consumes the data resolved by the promises and returns a new promise - * that will be resolved once all promises have been resolved. - * - * The promise is resolved even if one of the underlying collection - * promises is rejected. - */ - _handleCollectionPromises: function (promises, providerDiagnostic=null) { - return Task.spawn(function waitForPromises() { - for (let [name, promise] of promises) { - if (providerDiagnostic) { - providerDiagnostic(name); - } - - try { - yield promise; - this._log.debug("Provider collected successfully: " + name); - } catch (ex) { - this._recordProviderError(name, "Failed to collect", ex); - } - } - - throw new Task.Result(this); - }.bind(this)); - }, - - /** - * Record an error that occurred operating on a provider. - */ - _recordProviderError: function (name, msg, ex) { - msg = "Provider error: " + name + ": " + msg; - if (ex) { - msg += ": " + CommonUtils.exceptionStr(ex); - } - this._log.warn(msg); - - if (this.onProviderError) { - try { - this.onProviderError(msg); - } catch (callError) { - this._log.warn("Exception when calling onProviderError callback: " + - CommonUtils.exceptionStr(callError)); - } - } - }, -}); - diff --git a/services/metrics/storage.jsm b/services/metrics/storage.jsm deleted file mode 100644 index 9c98da18d2..0000000000 --- a/services/metrics/storage.jsm +++ /dev/null @@ -1,2189 +0,0 @@ -/* This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ - -#ifndef MERGED_COMPARTMENT - -"use strict"; - -this.EXPORTED_SYMBOLS = [ - "DailyValues", - "MetricsStorageBackend", - "dateToDays", - "daysToDate", -]; - -const {utils: Cu} = Components; - -const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000; - -#endif - -Cu.import("resource://gre/modules/Promise.jsm"); -Cu.import("resource://gre/modules/Sqlite.jsm"); -Cu.import("resource://gre/modules/AsyncShutdown.jsm"); -Cu.import("resource://gre/modules/Task.jsm"); -Cu.import("resource://gre/modules/Log.jsm"); -Cu.import("resource://services-common/utils.js"); - - -// These do not account for leap seconds. Meh. -function dateToDays(date) { - return Math.floor(date.getTime() / MILLISECONDS_PER_DAY); -} - -function daysToDate(days) { - return new Date(days * MILLISECONDS_PER_DAY); -} - -/** - * Represents a collection of per-day values. - * - * This is a proxy around a Map which can transparently round Date instances to - * their appropriate key. - * - * This emulates Map by providing .size and iterator support. Note that keys - * from the iterator are Date instances corresponding to midnight of the start - * of the day. get(), has(), and set() are modeled as getDay(), hasDay(), and - * setDay(), respectively. - * - * All days are defined in terms of UTC (as opposed to local time). - */ -this.DailyValues = function () { - this._days = new Map(); -}; - -DailyValues.prototype = Object.freeze({ - __iterator__: function () { - for (let [k, v] of this._days) { - yield [daysToDate(k), v]; - } - }, - - get size() { - return this._days.size; - }, - - hasDay: function (date) { - return this._days.has(dateToDays(date)); - }, - - getDay: function (date) { - return this._days.get(dateToDays(date)); - }, - - setDay: function (date, value) { - this._days.set(dateToDays(date), value); - }, - - appendValue: function (date, value) { - let key = dateToDays(date); - - if (this._days.has(key)) { - return this._days.get(key).push(value); - } - - this._days.set(key, [value]); - }, -}); - - -/** - * DATABASE INFO - * ============= - * - * We use a SQLite database as the backend for persistent storage of metrics - * data. - * - * Every piece of recorded data is associated with a measurement. A measurement - * is an entity with a name and version. Each measurement is associated with a - * named provider. - * - * When the metrics system is initialized, we ask providers (the entities that - * emit data) to configure the database for storage of their data. They tell - * storage what their requirements are. For example, they'll register - * named daily counters associated with specific measurements. - * - * Recorded data is stored differently depending on the requirements for - * storing it. We have facilities for storing the following classes of data: - * - * 1) Counts of event/field occurrences aggregated by day. - * 2) Discrete values of fields aggregated by day. - * 3) Discrete values of fields aggregated by day max 1 per day (last write - * wins). - * 4) Discrete values of fields max 1 (last write wins). - * - * Most data is aggregated per day mainly for privacy reasons. This does throw - * away potentially useful data. But, it's not currently used, so there is no - * need to keep the granular information. - * - * Database Schema - * --------------- - * - * This database contains the following tables: - * - * providers -- Maps provider string name to an internal ID. - * provider_state -- Holds opaque persisted state for providers. - * measurements -- Holds the set of known measurements (name, version, - * provider tuples). - * types -- The data types that can be stored in measurements/fields. - * fields -- Describes entities that occur within measurements. - * daily_counters -- Holds daily-aggregated counts of events. Each row is - * associated with a field and a day. - * daily_discrete_numeric -- Holds numeric values for fields grouped by day. - * Each row contains a discrete value associated with a field that occurred - * on a specific day. There can be multiple rows per field per day. - * daily_discrete_text -- Holds text values for fields grouped by day. Each - * row contains a discrete value associated with a field that occurred on a - * specific day. - * daily_last_numeric -- Holds numeric values where the last encountered - * value for a given day is retained. - * daily_last_text -- Like daily_last_numeric except for text values. - * last_numeric -- Holds the most recent value for a numeric field. - * last_text -- Like last_numeric except for text fields. - * - * Notes - * ----- - * - * It is tempting to use SQLite's julianday() function to store days that - * things happened. However, a Julian Day begins at *noon* in 4714 B.C. This - * results in weird half day offsets from UNIX time. So, we instead store - * number of days since UNIX epoch, not Julian. - */ - -/** - * All of our SQL statements are stored in a central mapping so they can easily - * be audited for security, perf, etc. - */ -const SQL = { - // Create the providers table. - createProvidersTable: "\ -CREATE TABLE providers (\ - id INTEGER PRIMARY KEY AUTOINCREMENT, \ - name TEXT, \ - UNIQUE (name) \ -)", - - createProviderStateTable: "\ -CREATE TABLE provider_state (\ - id INTEGER PRIMARY KEY AUTOINCREMENT, \ - provider_id INTEGER, \ - name TEXT, \ - VALUE TEXT, \ - UNIQUE (provider_id, name), \ - FOREIGN KEY (provider_id) REFERENCES providers(id) ON DELETE CASCADE\ -)", - - createProviderStateProviderIndex: "\ -CREATE INDEX i_provider_state_provider_id ON provider_state (provider_id)", - - createMeasurementsTable: "\ -CREATE TABLE measurements (\ - id INTEGER PRIMARY KEY AUTOINCREMENT, \ - provider_id INTEGER, \ - name TEXT, \ - version INTEGER, \ - UNIQUE (provider_id, name, version), \ - FOREIGN KEY (provider_id) REFERENCES providers(id) ON DELETE CASCADE\ -)", - - createMeasurementsProviderIndex: "\ -CREATE INDEX i_measurements_provider_id ON measurements (provider_id)", - - createMeasurementsView: "\ -CREATE VIEW v_measurements AS \ - SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version \ - FROM providers, measurements \ - WHERE \ - measurements.provider_id = providers.id", - - createTypesTable: "\ -CREATE TABLE types (\ - id INTEGER PRIMARY KEY AUTOINCREMENT, \ - name TEXT, \ - UNIQUE (name)\ -)", - - createFieldsTable: "\ -CREATE TABLE fields (\ - id INTEGER PRIMARY KEY AUTOINCREMENT, \ - measurement_id INTEGER, \ - name TEXT, \ - value_type INTEGER , \ - UNIQUE (measurement_id, name), \ - FOREIGN KEY (measurement_id) REFERENCES measurements(id) ON DELETE CASCADE \ - FOREIGN KEY (value_type) REFERENCES types(id) ON DELETE CASCADE \ -)", - - createFieldsMeasurementIndex: "\ -CREATE INDEX i_fields_measurement_id ON fields (measurement_id)", - - createFieldsView: "\ -CREATE VIEW v_fields AS \ - SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version, \ - fields.id AS field_id, \ - fields.name AS field_name, \ - types.id AS type_id, \ - types.name AS type_name \ - FROM providers, measurements, fields, types \ - WHERE \ - fields.measurement_id = measurements.id \ - AND measurements.provider_id = providers.id \ - AND fields.value_type = types.id", - - createDailyCountersTable: "\ -CREATE TABLE daily_counters (\ - field_id INTEGER, \ - day INTEGER, \ - value INTEGER, \ - UNIQUE(field_id, day), \ - FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\ -)", - - createDailyCountersFieldIndex: "\ -CREATE INDEX i_daily_counters_field_id ON daily_counters (field_id)", - - createDailyCountersDayIndex: "\ -CREATE INDEX i_daily_counters_day ON daily_counters (day)", - - createDailyCountersView: "\ -CREATE VIEW v_daily_counters AS SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version, \ - fields.id AS field_id, \ - fields.name AS field_name, \ - daily_counters.day AS day, \ - daily_counters.value AS value \ -FROM providers, measurements, fields, daily_counters \ -WHERE \ - daily_counters.field_id = fields.id \ - AND fields.measurement_id = measurements.id \ - AND measurements.provider_id = providers.id", - - createDailyDiscreteNumericsTable: "\ -CREATE TABLE daily_discrete_numeric (\ - id INTEGER PRIMARY KEY AUTOINCREMENT, \ - field_id INTEGER, \ - day INTEGER, \ - value INTEGER, \ - FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\ -)", - - createDailyDiscreteNumericsFieldIndex: "\ -CREATE INDEX i_daily_discrete_numeric_field_id \ -ON daily_discrete_numeric (field_id)", - - createDailyDiscreteNumericsDayIndex: "\ -CREATE INDEX i_daily_discrete_numeric_day \ -ON daily_discrete_numeric (day)", - - createDailyDiscreteTextTable: "\ -CREATE TABLE daily_discrete_text (\ - id INTEGER PRIMARY KEY AUTOINCREMENT, \ - field_id INTEGER, \ - day INTEGER, \ - value TEXT, \ - FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\ -)", - - createDailyDiscreteTextFieldIndex: "\ -CREATE INDEX i_daily_discrete_text_field_id \ -ON daily_discrete_text (field_id)", - - createDailyDiscreteTextDayIndex: "\ -CREATE INDEX i_daily_discrete_text_day \ -ON daily_discrete_text (day)", - - createDailyDiscreteView: "\ -CREATE VIEW v_daily_discrete AS \ - SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version, \ - fields.id AS field_id, \ - fields.name AS field_name, \ - daily_discrete_numeric.id AS value_id, \ - daily_discrete_numeric.day AS day, \ - daily_discrete_numeric.value AS value, \ - \"numeric\" AS value_type \ - FROM providers, measurements, fields, daily_discrete_numeric \ - WHERE \ - daily_discrete_numeric.field_id = fields.id \ - AND fields.measurement_id = measurements.id \ - AND measurements.provider_id = providers.id \ - UNION ALL \ - SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version, \ - fields.id AS field_id, \ - fields.name AS field_name, \ - daily_discrete_text.id AS value_id, \ - daily_discrete_text.day AS day, \ - daily_discrete_text.value AS value, \ - \"text\" AS value_type \ - FROM providers, measurements, fields, daily_discrete_text \ - WHERE \ - daily_discrete_text.field_id = fields.id \ - AND fields.measurement_id = measurements.id \ - AND measurements.provider_id = providers.id \ - ORDER BY day ASC, value_id ASC", - - createLastNumericTable: "\ -CREATE TABLE last_numeric (\ - field_id INTEGER PRIMARY KEY, \ - day INTEGER, \ - value NUMERIC, \ - FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\ -)", - - createLastTextTable: "\ -CREATE TABLE last_text (\ - field_id INTEGER PRIMARY KEY, \ - day INTEGER, \ - value TEXT, \ - FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\ -)", - - createLastView: "\ -CREATE VIEW v_last AS \ - SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version, \ - fields.id AS field_id, \ - fields.name AS field_name, \ - last_numeric.day AS day, \ - last_numeric.value AS value, \ - \"numeric\" AS value_type \ - FROM providers, measurements, fields, last_numeric \ - WHERE \ - last_numeric.field_id = fields.id \ - AND fields.measurement_id = measurements.id \ - AND measurements.provider_id = providers.id \ - UNION ALL \ - SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version, \ - fields.id AS field_id, \ - fields.name AS field_name, \ - last_text.day AS day, \ - last_text.value AS value, \ - \"text\" AS value_type \ - FROM providers, measurements, fields, last_text \ - WHERE \ - last_text.field_id = fields.id \ - AND fields.measurement_id = measurements.id \ - AND measurements.provider_id = providers.id", - - createDailyLastNumericTable: "\ -CREATE TABLE daily_last_numeric (\ - field_id INTEGER, \ - day INTEGER, \ - value NUMERIC, \ - UNIQUE (field_id, day) \ - FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\ -)", - - createDailyLastNumericFieldIndex: "\ -CREATE INDEX i_daily_last_numeric_field_id ON daily_last_numeric (field_id)", - - createDailyLastNumericDayIndex: "\ -CREATE INDEX i_daily_last_numeric_day ON daily_last_numeric (day)", - - createDailyLastTextTable: "\ -CREATE TABLE daily_last_text (\ - field_id INTEGER, \ - day INTEGER, \ - value TEXT, \ - UNIQUE (field_id, day) \ - FOREIGN KEY (field_id) REFERENCES fields(id) ON DELETE CASCADE\ -)", - - createDailyLastTextFieldIndex: "\ -CREATE INDEX i_daily_last_text_field_id ON daily_last_text (field_id)", - - createDailyLastTextDayIndex: "\ -CREATE INDEX i_daily_last_text_day ON daily_last_text (day)", - - createDailyLastView: "\ -CREATE VIEW v_daily_last AS \ - SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version, \ - fields.id AS field_id, \ - fields.name AS field_name, \ - daily_last_numeric.day AS day, \ - daily_last_numeric.value AS value, \ - \"numeric\" as value_type \ - FROM providers, measurements, fields, daily_last_numeric \ - WHERE \ - daily_last_numeric.field_id = fields.id \ - AND fields.measurement_id = measurements.id \ - AND measurements.provider_id = providers.id \ - UNION ALL \ - SELECT \ - providers.id AS provider_id, \ - providers.name AS provider_name, \ - measurements.id AS measurement_id, \ - measurements.name AS measurement_name, \ - measurements.version AS measurement_version, \ - fields.id AS field_id, \ - fields.name AS field_name, \ - daily_last_text.day AS day, \ - daily_last_text.value AS value, \ - \"text\" as value_type \ - FROM providers, measurements, fields, daily_last_text \ - WHERE \ - daily_last_text.field_id = fields.id \ - AND fields.measurement_id = measurements.id \ - AND measurements.provider_id = providers.id", - - // Mutation. - - addProvider: "INSERT INTO providers (name) VALUES (:provider)", - - setProviderState: "\ -INSERT OR REPLACE INTO provider_state \ - (provider_id, name, value) \ - VALUES (:provider_id, :name, :value)", - - addMeasurement: "\ -INSERT INTO measurements (provider_id, name, version) \ - VALUES (:provider_id, :measurement, :version)", - - addType: "INSERT INTO types (name) VALUES (:name)", - - addField: "\ -INSERT INTO fields (measurement_id, name, value_type) \ - VALUES (:measurement_id, :field, :value_type)", - - incrementDailyCounterFromFieldID: "\ -INSERT OR REPLACE INTO daily_counters VALUES (\ - :field_id, \ - :days, \ - COALESCE(\ - (SELECT value FROM daily_counters WHERE \ - field_id = :field_id AND day = :days \ - ), \ - 0\ - ) + :by)", - - deleteLastNumericFromFieldID: "\ -DELETE FROM last_numeric WHERE field_id = :field_id", - - deleteLastTextFromFieldID: "\ -DELETE FROM last_text WHERE field_id = :field_id", - - setLastNumeric: "\ -INSERT OR REPLACE INTO last_numeric VALUES (:field_id, :days, :value)", - - setLastText: "\ -INSERT OR REPLACE INTO last_text VALUES (:field_id, :days, :value)", - - setDailyLastNumeric: "\ -INSERT OR REPLACE INTO daily_last_numeric VALUES (:field_id, :days, :value)", - - setDailyLastText: "\ -INSERT OR REPLACE INTO daily_last_text VALUES (:field_id, :days, :value)", - - addDailyDiscreteNumeric: "\ -INSERT INTO daily_discrete_numeric \ -(field_id, day, value) VALUES (:field_id, :days, :value)", - - addDailyDiscreteText: "\ -INSERT INTO daily_discrete_text \ -(field_id, day, value) VALUES (:field_id, :days, :value)", - - pruneOldDailyCounters: "DELETE FROM daily_counters WHERE day < :days", - pruneOldDailyDiscreteNumeric: "DELETE FROM daily_discrete_numeric WHERE day < :days", - pruneOldDailyDiscreteText: "DELETE FROM daily_discrete_text WHERE day < :days", - pruneOldDailyLastNumeric: "DELETE FROM daily_last_numeric WHERE day < :days", - pruneOldDailyLastText: "DELETE FROM daily_last_text WHERE day < :days", - pruneOldLastNumeric: "DELETE FROM last_numeric WHERE day < :days", - pruneOldLastText: "DELETE FROM last_text WHERE day < :days", - - // Retrieval. - - getProviderID: "SELECT id FROM providers WHERE name = :provider", - - getProviders: "SELECT id, name FROM providers", - - getProviderStateWithName: "\ -SELECT value FROM provider_state \ - WHERE provider_id = :provider_id \ - AND name = :name", - - getMeasurements: "SELECT * FROM v_measurements", - - getMeasurementID: "\ -SELECT id FROM measurements \ - WHERE provider_id = :provider_id \ - AND name = :measurement \ - AND version = :version", - - getFieldID: "\ -SELECT id FROM fields \ - WHERE measurement_id = :measurement_id \ - AND name = :field \ - AND value_type = :value_type \ -", - - getTypes: "SELECT * FROM types", - - getTypeID: "SELECT id FROM types WHERE name = :name", - - getDailyCounterCountsFromFieldID: "\ -SELECT day, value FROM daily_counters \ - WHERE field_id = :field_id \ - ORDER BY day ASC", - - getDailyCounterCountFromFieldID: "\ -SELECT value FROM daily_counters \ - WHERE field_id = :field_id \ - AND day = :days", - - getMeasurementDailyCounters: "\ -SELECT field_name, day, value FROM v_daily_counters \ -WHERE measurement_id = :measurement_id", - - getFieldInfo: "SELECT * FROM v_fields", - - getLastNumericFromFieldID: "\ -SELECT day, value FROM last_numeric WHERE field_id = :field_id", - - getLastTextFromFieldID: "\ -SELECT day, value FROM last_text WHERE field_id = :field_id", - - getMeasurementLastValues: "\ -SELECT field_name, day, value FROM v_last \ -WHERE measurement_id = :measurement_id", - - getDailyDiscreteNumericFromFieldID: "\ -SELECT day, value FROM daily_discrete_numeric \ - WHERE field_id = :field_id \ - ORDER BY day ASC, id ASC", - - getDailyDiscreteNumericFromFieldIDAndDay: "\ -SELECT day, value FROM daily_discrete_numeric \ - WHERE field_id = :field_id AND day = :days \ - ORDER BY id ASC", - - getDailyDiscreteTextFromFieldID: "\ -SELECT day, value FROM daily_discrete_text \ - WHERE field_id = :field_id \ - ORDER BY day ASC, id ASC", - - getDailyDiscreteTextFromFieldIDAndDay: "\ -SELECT day, value FROM daily_discrete_text \ - WHERE field_id = :field_id AND day = :days \ - ORDER BY id ASC", - - getMeasurementDailyDiscreteValues: "\ -SELECT field_name, day, value_id, value FROM v_daily_discrete \ -WHERE measurement_id = :measurement_id \ -ORDER BY day ASC, value_id ASC", - - getDailyLastNumericFromFieldID: "\ -SELECT day, value FROM daily_last_numeric \ - WHERE field_id = :field_id \ - ORDER BY day ASC", - - getDailyLastNumericFromFieldIDAndDay: "\ -SELECT day, value FROM daily_last_numeric \ - WHERE field_id = :field_id AND day = :days", - - getDailyLastTextFromFieldID: "\ -SELECT day, value FROM daily_last_text \ - WHERE field_id = :field_id \ - ORDER BY day ASC", - - getDailyLastTextFromFieldIDAndDay: "\ -SELECT day, value FROM daily_last_text \ - WHERE field_id = :field_id AND day = :days", - - getMeasurementDailyLastValues: "\ -SELECT field_name, day, value FROM v_daily_last \ -WHERE measurement_id = :measurement_id", -}; - - -function dailyKeyFromDate(date) { - let year = String(date.getUTCFullYear()); - let month = String(date.getUTCMonth() + 1); - let day = String(date.getUTCDate()); - - if (month.length < 2) { - month = "0" + month; - } - - if (day.length < 2) { - day = "0" + day; - } - - return year + "-" + month + "-" + day; -} - - -/** - * Create a new backend instance bound to a SQLite database at the given path. - * - * This returns a promise that will resolve to a `MetricsStorageSqliteBackend` - * instance. The resolved instance will be initialized and ready for use. - * - * Very few consumers have a need to call this. Instead, a higher-level entity - * likely calls this and sets up the database connection for a service or - * singleton. - */ -this.MetricsStorageBackend = function (path) { - return Task.spawn(function initTask() { - let connection = yield Sqlite.openConnection({ - path: path, - - // There should only be one connection per database, so we disable this - // for perf reasons. - sharedMemoryCache: false, - }); - - // If we fail initializing the storage object, we need to close the - // database connection or else Storage will assert on shutdown. - let storage; - try { - storage = new MetricsStorageSqliteBackend(connection); - yield storage._init(); - } catch (ex) { - yield connection.close(); - throw ex; - } - - throw new Task.Result(storage); - }); -}; - -// Expose an asynchronous barrier `shutdown` that clients may use to -// perform last minute cleanup and shutdown requests before this module -// is shut down. -// See the documentation of AsyncShutdown.Barrier for more details. -var shutdown = new AsyncShutdown.Barrier("Metrics Storage Backend"); -this.MetricsStorageBackend.shutdown = shutdown.client; -Sqlite.shutdown.addBlocker("Metrics Storage: Shutting down", - () => shutdown.wait()); - -/** - * Manages storage of metrics data in a SQLite database. - * - * This is the main type used for interfacing with the database. - * - * Instances of this should be obtained by calling MetricsStorageConnection(). - * - * The current implementation will not work if the database is mutated by - * multiple connections because of the way we cache primary keys. - * - * FUTURE enforce 1 read/write connection per database limit. - */ -function MetricsStorageSqliteBackend(connection) { - this._log = Log.repository.getLogger("Services.Metrics.MetricsStorage"); - - this._connection = connection; - this._enabledWALCheckpointPages = null; - - // Integer IDs to string name. - this._typesByID = new Map(); - - // String name to integer IDs. - this._typesByName = new Map(); - - // Maps provider names to integer IDs. - this._providerIDs = new Map(); - - // Maps :-delimited strings of [provider name, name, version] to integer IDs. - this._measurementsByInfo = new Map(); - - // Integer IDs to Arrays of [provider name, name, version]. - this._measurementsByID = new Map(); - - // Integer IDs to Arrays of [measurement id, field name, value name] - this._fieldsByID = new Map(); - - // Maps :-delimited strings of [measurement id, field name] to integer ID. - this._fieldsByInfo = new Map(); - - // Maps measurement ID to sets of field IDs. - this._fieldsByMeasurement = new Map(); - - this._queuedOperations = []; - this._queuedInProgress = false; -} - -MetricsStorageSqliteBackend.prototype = Object.freeze({ - // Max size (in kibibytes) the WAL log is allowed to grow to before it is - // checkpointed. - // - // This was first deployed in bug 848136. We want a value large enough - // that we aren't checkpointing all the time. However, we want it - // small enough so we don't have to read so much when we open the - // database. - MAX_WAL_SIZE_KB: 512, - - FIELD_DAILY_COUNTER: "daily-counter", - FIELD_DAILY_DISCRETE_NUMERIC: "daily-discrete-numeric", - FIELD_DAILY_DISCRETE_TEXT: "daily-discrete-text", - FIELD_DAILY_LAST_NUMERIC: "daily-last-numeric", - FIELD_DAILY_LAST_TEXT: "daily-last-text", - FIELD_LAST_NUMERIC: "last-numeric", - FIELD_LAST_TEXT: "last-text", - - _BUILTIN_TYPES: [ - "FIELD_DAILY_COUNTER", - "FIELD_DAILY_DISCRETE_NUMERIC", - "FIELD_DAILY_DISCRETE_TEXT", - "FIELD_DAILY_LAST_NUMERIC", - "FIELD_DAILY_LAST_TEXT", - "FIELD_LAST_NUMERIC", - "FIELD_LAST_TEXT", - ], - - // Statements that are used to create the initial DB schema. - _SCHEMA_STATEMENTS: [ - "createProvidersTable", - "createProviderStateTable", - "createProviderStateProviderIndex", - "createMeasurementsTable", - "createMeasurementsProviderIndex", - "createMeasurementsView", - "createTypesTable", - "createFieldsTable", - "createFieldsMeasurementIndex", - "createFieldsView", - "createDailyCountersTable", - "createDailyCountersFieldIndex", - "createDailyCountersDayIndex", - "createDailyCountersView", - "createDailyDiscreteNumericsTable", - "createDailyDiscreteNumericsFieldIndex", - "createDailyDiscreteNumericsDayIndex", - "createDailyDiscreteTextTable", - "createDailyDiscreteTextFieldIndex", - "createDailyDiscreteTextDayIndex", - "createDailyDiscreteView", - "createDailyLastNumericTable", - "createDailyLastNumericFieldIndex", - "createDailyLastNumericDayIndex", - "createDailyLastTextTable", - "createDailyLastTextFieldIndex", - "createDailyLastTextDayIndex", - "createDailyLastView", - "createLastNumericTable", - "createLastTextTable", - "createLastView", - ], - - // Statements that are used to prune old data. - _PRUNE_STATEMENTS: [ - "pruneOldDailyCounters", - "pruneOldDailyDiscreteNumeric", - "pruneOldDailyDiscreteText", - "pruneOldDailyLastNumeric", - "pruneOldDailyLastText", - "pruneOldLastNumeric", - "pruneOldLastText", - ], - - /** - * Close the database connection. - * - * This should be called on all instances or the SQLite layer may complain - * loudly. After this has been called, the connection cannot be used. - * - * @return Promise<> A promise fulfilled once the connection is closed. - * This promise never rejects. - */ - close: function () { - return Task.spawn(function doClose() { - // There is some light magic involved here. First, we enqueue an - // operation to ensure that all pending operations have the opportunity - // to execute. We additionally execute a SQL operation. Due to the FIFO - // execution order of issued statements, this will cause us to wait on - // any outstanding statements before closing. - try { - yield this.enqueueOperation(function dummyOperation() { - return this._connection.execute("SELECT 1"); - }.bind(this)); - } catch (ex) {} - - try { - yield this._connection.close(); - } finally { - this._connection = null; - } - }.bind(this)); - }, - - /** - * Whether a provider is known to exist. - * - * @param provider - * (string) Name of the provider. - */ - hasProvider: function (provider) { - return this._providerIDs.has(provider); - }, - - /** - * Whether a measurement is known to exist. - * - * @param provider - * (string) Name of the provider. - * @param name - * (string) Name of the measurement. - * @param version - * (Number) Integer measurement version. - */ - hasMeasurement: function (provider, name, version) { - return this._measurementsByInfo.has([provider, name, version].join(":")); - }, - - /** - * Whether a named field exists in a measurement. - * - * @param measurementID - * (Number) The integer primary key of the measurement. - * @param field - * (string) The name of the field to look for. - */ - hasFieldFromMeasurement: function (measurementID, field) { - return this._fieldsByInfo.has([measurementID, field].join(":")); - }, - - /** - * Whether a field is known. - * - * @param provider - * (string) Name of the provider having the field. - * @param measurement - * (string) Name of the measurement in the provider having the field. - * @param field - * (string) Name of the field in the measurement. - */ - hasField: function (provider, measurement, version, field) { - let key = [provider, measurement, version].join(":"); - let measurementID = this._measurementsByInfo.get(key); - if (!measurementID) { - return false; - } - - return this.hasFieldFromMeasurement(measurementID, field); - }, - - /** - * Look up the integer primary key of a provider. - * - * @param provider - * (string) Name of the provider. - */ - providerID: function (provider) { - return this._providerIDs.get(provider); - }, - - /** - * Look up the integer primary key of a measurement. - * - * @param provider - * (string) Name of the provider. - * @param measurement - * (string) Name of the measurement. - * @param version - * (Number) Integer version of the measurement. - */ - measurementID: function (provider, measurement, version) { - return this._measurementsByInfo.get([provider, measurement, version].join(":")); - }, - - fieldIDFromMeasurement: function (measurementID, field) { - return this._fieldsByInfo.get([measurementID, field].join(":")); - }, - - fieldID: function (provider, measurement, version, field) { - let measurementID = this.measurementID(provider, measurement, version); - if (!measurementID) { - return null; - } - - return this.fieldIDFromMeasurement(measurementID, field); - }, - - measurementHasAnyDailyCounterFields: function (measurementID) { - return this.measurementHasAnyFieldsOfTypes(measurementID, - [this.FIELD_DAILY_COUNTER]); - }, - - measurementHasAnyLastFields: function (measurementID) { - return this.measurementHasAnyFieldsOfTypes(measurementID, - [this.FIELD_LAST_NUMERIC, - this.FIELD_LAST_TEXT]); - }, - - measurementHasAnyDailyLastFields: function (measurementID) { - return this.measurementHasAnyFieldsOfTypes(measurementID, - [this.FIELD_DAILY_LAST_NUMERIC, - this.FIELD_DAILY_LAST_TEXT]); - }, - - measurementHasAnyDailyDiscreteFields: function (measurementID) { - return this.measurementHasAnyFieldsOfTypes(measurementID, - [this.FIELD_DAILY_DISCRETE_NUMERIC, - this.FIELD_DAILY_DISCRETE_TEXT]); - }, - - measurementHasAnyFieldsOfTypes: function (measurementID, types) { - if (!this._fieldsByMeasurement.has(measurementID)) { - return false; - } - - let fieldIDs = this._fieldsByMeasurement.get(measurementID); - for (let fieldID of fieldIDs) { - let fieldType = this._fieldsByID.get(fieldID)[2]; - if (types.indexOf(fieldType) != -1) { - return true; - } - } - - return false; - }, - - /** - * Register a measurement with the backend. - * - * Measurements must be registered before storage can be allocated to them. - * - * A measurement consists of a string name and integer version attached - * to a named provider. - * - * This returns a promise that resolves to the storage ID for this - * measurement. - * - * If the measurement is not known to exist, it is registered with storage. - * If the measurement has already been registered, this is effectively a - * no-op (that still returns a promise resolving to the storage ID). - * - * @param provider - * (string) Name of the provider this measurement belongs to. - * @param name - * (string) Name of this measurement. - * @param version - * (Number) Integer version of this measurement. - */ - registerMeasurement: function (provider, name, version) { - if (this.hasMeasurement(provider, name, version)) { - return CommonUtils.laterTickResolvingPromise( - this.measurementID(provider, name, version)); - } - - // Registrations might not be safe to perform in parallel with provider - // operations. So, we queue them. - let self = this; - return this.enqueueOperation(function createMeasurementOperation() { - return Task.spawn(function createMeasurement() { - let providerID = self._providerIDs.get(provider); - - if (!providerID) { - yield self._connection.executeCached(SQL.addProvider, {provider: provider}); - let rows = yield self._connection.executeCached(SQL.getProviderID, - {provider: provider}); - - providerID = rows[0].getResultByIndex(0); - - self._providerIDs.set(provider, providerID); - } - - let params = { - provider_id: providerID, - measurement: name, - version: version, - }; - - yield self._connection.executeCached(SQL.addMeasurement, params); - let rows = yield self._connection.executeCached(SQL.getMeasurementID, params); - - let measurementID = rows[0].getResultByIndex(0); - - self._measurementsByInfo.set([provider, name, version].join(":"), measurementID); - self._measurementsByID.set(measurementID, [provider, name, version]); - self._fieldsByMeasurement.set(measurementID, new Set()); - - throw new Task.Result(measurementID); - }); - }); - }, - - /** - * Register a field with the backend. - * - * Fields are what recorded pieces of data are primarily associated with. - * - * Fields are associated with measurements. Measurements must be registered - * via `registerMeasurement` before fields can be registered. This is - * enforced by this function requiring the database primary key of the - * measurement as an argument. - * - * @param measurementID - * (Number) Integer primary key of measurement this field belongs to. - * @param field - * (string) Name of this field. - * @param valueType - * (string) Type name of this field. Must be a registered type. Is - * likely one of the FIELD_ constants on this type. - * - * @return Promise - */ - registerField: function (measurementID, field, valueType) { - if (!valueType) { - throw new Error("Value type must be defined."); - } - - if (!this._measurementsByID.has(measurementID)) { - throw new Error("Measurement not known: " + measurementID); - } - - if (!this._typesByName.has(valueType)) { - throw new Error("Unknown value type: " + valueType); - } - - let typeID = this._typesByName.get(valueType); - - if (!typeID) { - throw new Error("Undefined type: " + valueType); - } - - if (this.hasFieldFromMeasurement(measurementID, field)) { - let id = this.fieldIDFromMeasurement(measurementID, field); - let existingType = this._fieldsByID.get(id)[2]; - - if (valueType != existingType) { - throw new Error("Field already defined with different type: " + existingType); - } - - return CommonUtils.laterTickResolvingPromise( - this.fieldIDFromMeasurement(measurementID, field)); - } - - let self = this; - return Task.spawn(function createField() { - let params = { - measurement_id: measurementID, - field: field, - value_type: typeID, - }; - - yield self._connection.executeCached(SQL.addField, params); - - let rows = yield self._connection.executeCached(SQL.getFieldID, params); - - let fieldID = rows[0].getResultByIndex(0); - - self._fieldsByID.set(fieldID, [measurementID, field, valueType]); - self._fieldsByInfo.set([measurementID, field].join(":"), fieldID); - self._fieldsByMeasurement.get(measurementID).add(fieldID); - - throw new Task.Result(fieldID); - }); - }, - - /** - * Initializes this instance with the database. - * - * This performs 2 major roles: - * - * 1) Set up database schema (creates tables). - * 2) Synchronize database with local instance. - */ - _init: function() { - let self = this; - return Task.spawn(function initTask() { - // 0. Database file and connection configuration. - - // This should never fail. But, we assume the default of 1024 in case it - // does. - let rows = yield self._connection.execute("PRAGMA page_size"); - let pageSize = 1024; - if (rows.length) { - pageSize = rows[0].getResultByIndex(0); - } - - self._log.debug("Page size is " + pageSize); - - // Ensure temp tables are stored in memory, not on disk. - yield self._connection.execute("PRAGMA temp_store=MEMORY"); - - let journalMode; - rows = yield self._connection.execute("PRAGMA journal_mode=WAL"); - if (rows.length) { - journalMode = rows[0].getResultByIndex(0); - } - - self._log.info("Journal mode is " + journalMode); - - if (journalMode == "wal") { - self._enabledWALCheckpointPages = - Math.ceil(self.MAX_WAL_SIZE_KB * 1024 / pageSize); - - self._log.info("WAL auto checkpoint pages: " + - self._enabledWALCheckpointPages); - - // We disable auto checkpoint during initialization to make it - // quicker. - yield self.setAutoCheckpoint(0); - } else { - if (journalMode != "truncate") { - // Fall back to truncate (which is faster than delete). - yield self._connection.execute("PRAGMA journal_mode=TRUNCATE"); - } - - // And always use full synchronous mode to reduce possibility for data - // loss. - yield self._connection.execute("PRAGMA synchronous=FULL"); - } - - let doCheckpoint = false; - - // 1. Create the schema. - yield self._connection.executeTransaction(function ensureSchema(conn) { - let schema = yield conn.getSchemaVersion(); - - if (schema == 0) { - self._log.info("Creating database schema."); - - for (let k of self._SCHEMA_STATEMENTS) { - yield self._connection.execute(SQL[k]); - } - - yield self._connection.setSchemaVersion(1); - doCheckpoint = true; - } else if (schema != 1) { - throw new Error("Unknown database schema: " + schema); - } else { - self._log.debug("Database schema up to date."); - } - }); - - // 2. Retrieve existing types. - yield self._connection.execute(SQL.getTypes, null, function onRow(row) { - let id = row.getResultByName("id"); - let name = row.getResultByName("name"); - - self._typesByID.set(id, name); - self._typesByName.set(name, id); - }); - - // 3. Populate built-in types with database. - let missingTypes = []; - for (let type of self._BUILTIN_TYPES) { - type = self[type]; - if (self._typesByName.has(type)) { - continue; - } - - missingTypes.push(type); - } - - // Don't perform DB transaction unless there is work to do. - if (missingTypes.length) { - yield self._connection.executeTransaction(function populateBuiltinTypes() { - for (let type of missingTypes) { - let params = {name: type}; - yield self._connection.executeCached(SQL.addType, params); - let rows = yield self._connection.executeCached(SQL.getTypeID, params); - let id = rows[0].getResultByIndex(0); - - self._typesByID.set(id, type); - self._typesByName.set(type, id); - } - }); - - doCheckpoint = true; - } - - // 4. Obtain measurement info. - yield self._connection.execute(SQL.getMeasurements, null, function onRow(row) { - let providerID = row.getResultByName("provider_id"); - let providerName = row.getResultByName("provider_name"); - let measurementID = row.getResultByName("measurement_id"); - let measurementName = row.getResultByName("measurement_name"); - let measurementVersion = row.getResultByName("measurement_version"); - - self._providerIDs.set(providerName, providerID); - - let info = [providerName, measurementName, measurementVersion].join(":"); - - self._measurementsByInfo.set(info, measurementID); - self._measurementsByID.set(measurementID, info); - self._fieldsByMeasurement.set(measurementID, new Set()); - }); - - // 5. Obtain field info. - yield self._connection.execute(SQL.getFieldInfo, null, function onRow(row) { - let measurementID = row.getResultByName("measurement_id"); - let fieldID = row.getResultByName("field_id"); - let fieldName = row.getResultByName("field_name"); - let typeName = row.getResultByName("type_name"); - - self._fieldsByID.set(fieldID, [measurementID, fieldName, typeName]); - self._fieldsByInfo.set([measurementID, fieldName].join(":"), fieldID); - self._fieldsByMeasurement.get(measurementID).add(fieldID); - }); - - // Perform a checkpoint after initialization (if needed) and - // enable auto checkpoint during regular operation. - if (doCheckpoint) { - yield self.checkpoint(); - } - - yield self.setAutoCheckpoint(1); - }); - }, - - /** - * Prune all data from earlier than the specified date. - * - * Data stored on days before the specified Date will be permanently - * deleted. - * - * This returns a promise that will be resolved when data has been deleted. - * - * @param date - * (Date) Old data threshold. - * @return Promise<> - */ - pruneDataBefore: function (date) { - let statements = this._PRUNE_STATEMENTS; - - let self = this; - return this.enqueueOperation(function doPrune() { - return self._connection.executeTransaction(function prune(conn) { - let days = dateToDays(date); - - let params = {days: days}; - for (let name of statements) { - yield conn.execute(SQL[name], params); - } - }); - }); - }, - - /** - * Reduce memory usage as much as possible. - * - * This returns a promise that will be resolved on completion. - * - * @return Promise<> - */ - compact: function () { - let self = this; - return this.enqueueOperation(function doCompact() { - self._connection.discardCachedStatements(); - return self._connection.shrinkMemory(); - }); - }, - - /** - * Checkpoint writes requiring flush to disk. - * - * This is called to persist queued and non-flushed writes to disk. - * It will force an fsync, so it is expensive and should be used - * sparingly. - */ - checkpoint: function () { - if (!this._enabledWALCheckpointPages) { - return CommonUtils.laterTickResolvingPromise(); - } - - return this.enqueueOperation(function checkpoint() { - this._log.info("Performing manual WAL checkpoint."); - return this._connection.execute("PRAGMA wal_checkpoint"); - }.bind(this)); - }, - - setAutoCheckpoint: function (on) { - // If we aren't in WAL mode, wal_autocheckpoint won't do anything so - // we no-op. - if (!this._enabledWALCheckpointPages) { - return CommonUtils.laterTickResolvingPromise(); - } - - let val = on ? this._enabledWALCheckpointPages : 0; - - return this.enqueueOperation(function setWALCheckpoint() { - this._log.info("Setting WAL auto checkpoint to " + val); - return this._connection.execute("PRAGMA wal_autocheckpoint=" + val); - }.bind(this)); - }, - - /** - * Ensure a field ID matches a specified type. - * - * This is called internally as part of adding values to ensure that - * the type of a field matches the operation being performed. - */ - _ensureFieldType: function (id, type) { - let info = this._fieldsByID.get(id); - - if (!info || !Array.isArray(info)) { - throw new Error("Unknown field ID: " + id); - } - - if (type != info[2]) { - throw new Error("Field type does not match the expected for this " + - "operation. Actual: " + info[2] + "; Expected: " + - type); - } - }, - - /** - * Enqueue a storage operation to be performed when the database is ready. - * - * The primary use case of this function is to prevent potentially - * conflicting storage operations from being performed in parallel. By - * calling this function, passed storage operations will be serially - * executed, avoiding potential order of operation issues. - * - * The passed argument is a function that will perform storage operations. - * The function should return a promise that will be resolved when all - * storage operations have been completed. - * - * The passed function may be executed immediately. If there are already - * queued operations, it will be appended to the queue and executed after all - * before it have finished. - * - * This function returns a promise that will be resolved or rejected with - * the same value that the function's promise was resolved or rejected with. - * - * @param func - * (function) Function performing storage interactions. - * @return Promise<> - */ - enqueueOperation: function (func) { - if (typeof(func) != "function") { - throw new Error("enqueueOperation expects a function. Got: " + typeof(func)); - } - - this._log.trace("Enqueueing operation."); - let deferred = Promise.defer(); - - this._queuedOperations.push([func, deferred]); - - if (this._queuedOperations.length == 1) { - this._popAndPerformQueuedOperation(); - } - - return deferred.promise; - }, - - /** - * Enqueue a function to be performed as a transaction. - * - * The passed function should be a generator suitable for calling with - * `executeTransaction` from the SQLite connection. - */ - enqueueTransaction: function (func, type) { - return this.enqueueOperation( - this._connection.executeTransaction.bind(this._connection, func, type) - ); - }, - - _popAndPerformQueuedOperation: function () { - if (!this._queuedOperations.length || this._queuedInProgress) { - return; - } - - this._log.trace("Performing queued operation."); - let [func, deferred] = this._queuedOperations.shift(); - let promise; - - try { - this._queuedInProgress = true; - promise = func(); - } catch (ex) { - this._log.warn("Queued operation threw during execution: " + - CommonUtils.exceptionStr(ex)); - this._queuedInProgress = false; - deferred.reject(ex); - this._popAndPerformQueuedOperation(); - return; - } - - if (!promise || typeof(promise.then) != "function") { - let msg = "Queued operation did not return a promise: " + func; - this._log.warn(msg); - - this._queuedInProgress = false; - deferred.reject(new Error(msg)); - this._popAndPerformQueuedOperation(); - return; - } - - promise.then( - function onSuccess(result) { - this._log.trace("Queued operation completed."); - this._queuedInProgress = false; - deferred.resolve(result); - this._popAndPerformQueuedOperation(); - }.bind(this), - function onError(error) { - this._log.warn("Failure when performing queued operation: " + - CommonUtils.exceptionStr(error)); - this._queuedInProgress = false; - deferred.reject(error); - this._popAndPerformQueuedOperation(); - }.bind(this) - ); - }, - - /** - * Obtain all values associated with a measurement. - * - * This returns a promise that resolves to an object. The keys of the object - * are: - * - * days -- DailyValues where the values are Maps of field name to data - * structures. The data structures could be simple (string or number) or - * Arrays if the field type allows multiple values per day. - * - * singular -- Map of field names to values. This holds all fields that - * don't have a temporal component. - * - * @param id - * (Number) Primary key of measurement whose values to retrieve. - */ - getMeasurementValues: function (id) { - let deferred = Promise.defer(); - let days = new DailyValues(); - let singular = new Map(); - - let self = this; - this.enqueueOperation(function enqueuedGetMeasurementValues() { - return Task.spawn(function fetchMeasurementValues() { - function handleResult(data) { - for (let [field, values] of data) { - for (let [day, value] of Iterator(values)) { - if (!days.hasDay(day)) { - days.setDay(day, new Map()); - } - - days.getDay(day).set(field, value); - } - } - } - - if (self.measurementHasAnyDailyCounterFields(id)) { - let counters = yield self.getMeasurementDailyCountersFromMeasurementID(id); - handleResult(counters); - } - - if (self.measurementHasAnyDailyLastFields(id)) { - let dailyLast = yield self.getMeasurementDailyLastValuesFromMeasurementID(id); - handleResult(dailyLast); - } - - if (self.measurementHasAnyDailyDiscreteFields(id)) { - let dailyDiscrete = yield self.getMeasurementDailyDiscreteValuesFromMeasurementID(id); - handleResult(dailyDiscrete); - } - - if (self.measurementHasAnyLastFields(id)) { - let last = yield self.getMeasurementLastValuesFromMeasurementID(id); - - for (let [field, value] of last) { - singular.set(field, value); - } - } - - }); - }).then(function onSuccess() { - deferred.resolve({singular: singular, days: days}); - }, function onError(error) { - deferred.reject(error); - }); - - return deferred.promise; - }, - - //--------------------------------------------------------------------------- - // Low-level storage operations - // - // These will be performed immediately (or at least as soon as the underlying - // connection allows them to be.) It is recommended to call these from within - // a function added via `enqueueOperation()` or they may inadvertently be - // performed during another enqueued operation, which may be a transaction - // that is rolled back. - // --------------------------------------------------------------------------- - - /** - * Set state for a provider. - * - * Providers have the ability to register persistent state with the backend. - * Persistent state doesn't expire. The format of the data is completely up - * to the provider beyond the requirement that values be UTF-8 strings. - * - * This returns a promise that will be resolved when the underlying database - * operation has completed. - * - * @param provider - * (string) Name of the provider. - * @param key - * (string) Key under which to store this state. - * @param value - * (string) Value for this state. - * @return Promise<> - */ - setProviderState: function (provider, key, value) { - if (typeof(key) != "string") { - throw new Error("State key must be a string. Got: " + key); - } - - if (typeof(value) != "string") { - throw new Error("State value must be a string. Got: " + value); - } - - let id = this.providerID(provider); - if (!id) { - throw new Error("Unknown provider: " + provider); - } - - return this._connection.executeCached(SQL.setProviderState, { - provider_id: id, - name: key, - value: value, - }); - }, - - /** - * Obtain named state for a provider. - * - * - * The returned promise will resolve to the state from the database or null - * if the key is not stored. - * - * @param provider - * (string) The name of the provider whose state to obtain. - * @param key - * (string) The state's key to retrieve. - * - * @return Promise - */ - getProviderState: function (provider, key) { - let id = this.providerID(provider); - if (!id) { - throw new Error("Unknown provider: " + provider); - } - - let conn = this._connection; - return Task.spawn(function queryDB() { - let rows = yield conn.executeCached(SQL.getProviderStateWithName, { - provider_id: id, - name: key, - }); - - if (!rows.length) { - throw new Task.Result(null); - } - - throw new Task.Result(rows[0].getResultByIndex(0)); - }); - }, - - /** - * Increment a daily counter from a numeric field id. - * - * @param id - * (integer) Primary key of field to increment. - * @param date - * (Date) When the increment occurred. This is typically "now" but can - * be explicitly defined for events that occurred in the past. - * @param by - * (integer) How much to increment the value by. Defaults to 1. - */ - incrementDailyCounterFromFieldID: function (id, date=new Date(), by=1) { - this._ensureFieldType(id, this.FIELD_DAILY_COUNTER); - - let params = { - field_id: id, - days: dateToDays(date), - by: by, - }; - - return this._connection.executeCached(SQL.incrementDailyCounterFromFieldID, - params); - }, - - /** - * Obtain all counts for a specific daily counter. - * - * @param id - * (integer) The ID of the field being retrieved. - */ - getDailyCounterCountsFromFieldID: function (id) { - this._ensureFieldType(id, this.FIELD_DAILY_COUNTER); - - let self = this; - return Task.spawn(function fetchCounterDays() { - let rows = yield self._connection.executeCached(SQL.getDailyCounterCountsFromFieldID, - {field_id: id}); - - let result = new DailyValues(); - for (let row of rows) { - let days = row.getResultByIndex(0); - let counter = row.getResultByIndex(1); - - let date = daysToDate(days); - result.setDay(date, counter); - } - - throw new Task.Result(result); - }); - }, - - /** - * Get the value of a daily counter for a given day. - * - * @param field - * (integer) Field ID to retrieve. - * @param date - * (Date) Date for day from which to obtain data. - */ - getDailyCounterCountFromFieldID: function (field, date) { - this._ensureFieldType(field, this.FIELD_DAILY_COUNTER); - - let params = { - field_id: field, - days: dateToDays(date), - }; - - let self = this; - return Task.spawn(function fetchCounter() { - let rows = yield self._connection.executeCached(SQL.getDailyCounterCountFromFieldID, - params); - if (!rows.length) { - throw new Task.Result(null); - } - - throw new Task.Result(rows[0].getResultByIndex(0)); - }); - }, - - /** - * Define the value for a "last numeric" field. - * - * The previous value (if any) will be replaced by the value passed, even if - * the date of the incoming value is older than what's recorded in the - * database. - * - * @param fieldID - * (Number) Integer primary key of field to update. - * @param value - * (Number) Value to record. - * @param date - * (Date) When this value was produced. - */ - setLastNumericFromFieldID: function (fieldID, value, date=new Date()) { - this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC); - - if (typeof(value) != "number") { - throw new Error("Value is not a number: " + value); - } - - let params = { - field_id: fieldID, - days: dateToDays(date), - value: value, - }; - - return this._connection.executeCached(SQL.setLastNumeric, params); - }, - - /** - * Define the value of a "last text" field. - * - * See `setLastNumericFromFieldID` for behavior. - */ - setLastTextFromFieldID: function (fieldID, value, date=new Date()) { - this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT); - - if (typeof(value) != "string") { - throw new Error("Value is not a string: " + value); - } - - let params = { - field_id: fieldID, - days: dateToDays(date), - value: value, - }; - - return this._connection.executeCached(SQL.setLastText, params); - }, - - /** - * Obtain the value of a "last numeric" field. - * - * This returns a promise that will be resolved with an Array of [date, value] - * if a value is known or null if no last value is present. - * - * @param fieldID - * (Number) Integer primary key of field to retrieve. - */ - getLastNumericFromFieldID: function (fieldID) { - this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC); - - let self = this; - return Task.spawn(function fetchLastField() { - let rows = yield self._connection.executeCached(SQL.getLastNumericFromFieldID, - {field_id: fieldID}); - - if (!rows.length) { - throw new Task.Result(null); - } - - let row = rows[0]; - let days = row.getResultByIndex(0); - let value = row.getResultByIndex(1); - - throw new Task.Result([daysToDate(days), value]); - }); - }, - - /** - * Obtain the value of a "last text" field. - * - * See `getLastNumericFromFieldID` for behavior. - */ - getLastTextFromFieldID: function (fieldID) { - this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT); - - let self = this; - return Task.spawn(function fetchLastField() { - let rows = yield self._connection.executeCached(SQL.getLastTextFromFieldID, - {field_id: fieldID}); - - if (!rows.length) { - throw new Task.Result(null); - } - - let row = rows[0]; - let days = row.getResultByIndex(0); - let value = row.getResultByIndex(1); - - throw new Task.Result([daysToDate(days), value]); - }); - }, - - /** - * Delete the value (if any) in a "last numeric" field. - */ - deleteLastNumericFromFieldID: function (fieldID) { - this._ensureFieldType(fieldID, this.FIELD_LAST_NUMERIC); - - return this._connection.executeCached(SQL.deleteLastNumericFromFieldID, - {field_id: fieldID}); - }, - - /** - * Delete the value (if any) in a "last text" field. - */ - deleteLastTextFromFieldID: function (fieldID) { - this._ensureFieldType(fieldID, this.FIELD_LAST_TEXT); - - return this._connection.executeCached(SQL.deleteLastTextFromFieldID, - {field_id: fieldID}); - }, - - /** - * Record a value for a "daily last numeric" field. - * - * The field can hold 1 value per calendar day. If the field already has a - * value for the day specified (defaults to now), that value will be - * replaced, even if the date specified is older (within the day) than the - * previously recorded value. - * - * @param fieldID - * (Number) Integer primary key of field. - * @param value - * (Number) Value to record. - * @param date - * (Date) When the value was produced. Defaults to now. - */ - setDailyLastNumericFromFieldID: function (fieldID, value, date=new Date()) { - this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_NUMERIC); - - let params = { - field_id: fieldID, - days: dateToDays(date), - value: value, - }; - - return this._connection.executeCached(SQL.setDailyLastNumeric, params); - }, - - /** - * Record a value for a "daily last text" field. - * - * See `setDailyLastNumericFromFieldID` for behavior. - */ - setDailyLastTextFromFieldID: function (fieldID, value, date=new Date()) { - this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_TEXT); - - let params = { - field_id: fieldID, - days: dateToDays(date), - value: value, - }; - - return this._connection.executeCached(SQL.setDailyLastText, params); - }, - - /** - * Obtain value(s) from a "daily last numeric" field. - * - * This returns a promise that resolves to a DailyValues instance. If `date` - * is specified, that instance will have at most 1 entry. If there is no - * `date` constraint, then all stored values will be retrieved. - * - * @param fieldID - * (Number) Integer primary key of field to retrieve. - * @param date optional - * (Date) If specified, only return data for this day. - * - * @return Promise - */ - getDailyLastNumericFromFieldID: function (fieldID, date=null) { - this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_NUMERIC); - - let params = {field_id: fieldID}; - let name = "getDailyLastNumericFromFieldID"; - - if (date) { - params.days = dateToDays(date); - name = "getDailyLastNumericFromFieldIDAndDay"; - } - - return this._getDailyLastFromFieldID(name, params); - }, - - /** - * Obtain value(s) from a "daily last text" field. - * - * See `getDailyLastNumericFromFieldID` for behavior. - */ - getDailyLastTextFromFieldID: function (fieldID, date=null) { - this._ensureFieldType(fieldID, this.FIELD_DAILY_LAST_TEXT); - - let params = {field_id: fieldID}; - let name = "getDailyLastTextFromFieldID"; - - if (date) { - params.days = dateToDays(date); - name = "getDailyLastTextFromFieldIDAndDay"; - } - - return this._getDailyLastFromFieldID(name, params); - }, - - _getDailyLastFromFieldID: function (name, params) { - let self = this; - return Task.spawn(function fetchDailyLastForField() { - let rows = yield self._connection.executeCached(SQL[name], params); - - let result = new DailyValues(); - for (let row of rows) { - let d = daysToDate(row.getResultByIndex(0)); - let value = row.getResultByIndex(1); - - result.setDay(d, value); - } - - throw new Task.Result(result); - }); - }, - - /** - * Add a new value for a "daily discrete numeric" field. - * - * This appends a new value to the list of values for a specific field. All - * values are retained. Duplicate values are allowed. - * - * @param fieldID - * (Number) Integer primary key of field. - * @param value - * (Number) Value to record. - * @param date optional - * (Date) When this value occurred. Values are bucketed by day. - */ - addDailyDiscreteNumericFromFieldID: function (fieldID, value, date=new Date()) { - this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_NUMERIC); - - if (typeof(value) != "number") { - throw new Error("Number expected. Got: " + value); - } - - let params = { - field_id: fieldID, - days: dateToDays(date), - value: value, - }; - - return this._connection.executeCached(SQL.addDailyDiscreteNumeric, params); - }, - - /** - * Add a new value for a "daily discrete text" field. - * - * See `addDailyDiscreteNumericFromFieldID` for behavior. - */ - addDailyDiscreteTextFromFieldID: function (fieldID, value, date=new Date()) { - this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_TEXT); - - if (typeof(value) != "string") { - throw new Error("String expected. Got: " + value); - } - - let params = { - field_id: fieldID, - days: dateToDays(date), - value: value, - }; - - return this._connection.executeCached(SQL.addDailyDiscreteText, params); - }, - - /** - * Obtain values for a "daily discrete numeric" field. - * - * This returns a promise that resolves to a `DailyValues` instance. If - * `date` is specified, there will be at most 1 key in that instance. If - * not, all data from the database will be retrieved. - * - * Values in that instance will be arrays of the raw values. - * - * @param fieldID - * (Number) Integer primary key of field to retrieve. - * @param date optional - * (Date) Day to obtain data for. Date can be any time in the day. - */ - getDailyDiscreteNumericFromFieldID: function (fieldID, date=null) { - this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_NUMERIC); - - let params = {field_id: fieldID}; - - let name = "getDailyDiscreteNumericFromFieldID"; - - if (date) { - params.days = dateToDays(date); - name = "getDailyDiscreteNumericFromFieldIDAndDay"; - } - - return this._getDailyDiscreteFromFieldID(name, params); - }, - - /** - * Obtain values for a "daily discrete text" field. - * - * See `getDailyDiscreteNumericFromFieldID` for behavior. - */ - getDailyDiscreteTextFromFieldID: function (fieldID, date=null) { - this._ensureFieldType(fieldID, this.FIELD_DAILY_DISCRETE_TEXT); - - let params = {field_id: fieldID}; - - let name = "getDailyDiscreteTextFromFieldID"; - - if (date) { - params.days = dateToDays(date); - name = "getDailyDiscreteTextFromFieldIDAndDay"; - } - - return this._getDailyDiscreteFromFieldID(name, params); - }, - - _getDailyDiscreteFromFieldID: function (name, params) { - let self = this; - return Task.spawn(function fetchDailyDiscreteValuesForField() { - let rows = yield self._connection.executeCached(SQL[name], params); - - let result = new DailyValues(); - for (let row of rows) { - let d = daysToDate(row.getResultByIndex(0)); - let value = row.getResultByIndex(1); - - result.appendValue(d, value); - } - - throw new Task.Result(result); - }); - }, - - /** - * Obtain the counts of daily counters in a measurement. - * - * This returns a promise that resolves to a Map of field name strings to - * DailyValues that hold per-day counts. - * - * @param id - * (Number) Integer primary key of measurement. - * - * @return Promise - */ - getMeasurementDailyCountersFromMeasurementID: function (id) { - let self = this; - return Task.spawn(function fetchDailyCounters() { - let rows = yield self._connection.execute(SQL.getMeasurementDailyCounters, - {measurement_id: id}); - - let result = new Map(); - for (let row of rows) { - let field = row.getResultByName("field_name"); - let date = daysToDate(row.getResultByName("day")); - let value = row.getResultByName("value"); - - if (!result.has(field)) { - result.set(field, new DailyValues()); - } - - result.get(field).setDay(date, value); - } - - throw new Task.Result(result); - }); - }, - - /** - * Obtain the values of "last" fields from a measurement. - * - * This returns a promise that resolves to a Map of field name to an array - * of [date, value]. - * - * @param id - * (Number) Integer primary key of measurement whose data to retrieve. - * - * @return Promise - */ - getMeasurementLastValuesFromMeasurementID: function (id) { - let self = this; - return Task.spawn(function fetchMeasurementLastValues() { - let rows = yield self._connection.execute(SQL.getMeasurementLastValues, - {measurement_id: id}); - - let result = new Map(); - for (let row of rows) { - let date = daysToDate(row.getResultByIndex(1)); - let value = row.getResultByIndex(2); - result.set(row.getResultByIndex(0), [date, value]); - } - - throw new Task.Result(result); - }); - }, - - /** - * Obtain the values of "last daily" fields from a measurement. - * - * This returns a promise that resolves to a Map of field name to DailyValues - * instances. Each DailyValues instance has days for which a daily last value - * is defined. The values in each DailyValues are the raw last value for that - * day. - * - * @param id - * (Number) Integer primary key of measurement whose data to retrieve. - * - * @return Promise - */ - getMeasurementDailyLastValuesFromMeasurementID: function (id) { - let self = this; - return Task.spawn(function fetchMeasurementDailyLastValues() { - let rows = yield self._connection.execute(SQL.getMeasurementDailyLastValues, - {measurement_id: id}); - - let result = new Map(); - for (let row of rows) { - let field = row.getResultByName("field_name"); - let date = daysToDate(row.getResultByName("day")); - let value = row.getResultByName("value"); - - if (!result.has(field)) { - result.set(field, new DailyValues()); - } - - result.get(field).setDay(date, value); - } - - throw new Task.Result(result); - }); - }, - - /** - * Obtain the values of "daily discrete" fields from a measurement. - * - * This obtains all discrete values for all "daily discrete" fields in a - * measurement. - * - * This returns a promise that resolves to a Map. The Map's keys are field - * string names. Values are `DailyValues` instances. The values inside - * the `DailyValues` are arrays of the raw discrete values. - * - * @param id - * (Number) Integer primary key of measurement. - * - * @return Promise - */ - getMeasurementDailyDiscreteValuesFromMeasurementID: function (id) { - let deferred = Promise.defer(); - let result = new Map(); - - this._connection.execute(SQL.getMeasurementDailyDiscreteValues, - {measurement_id: id}, function onRow(row) { - let field = row.getResultByName("field_name"); - let date = daysToDate(row.getResultByName("day")); - let value = row.getResultByName("value"); - - if (!result.has(field)) { - result.set(field, new DailyValues()); - } - - result.get(field).appendValue(date, value); - }).then(function onComplete() { - deferred.resolve(result); - }, function onError(error) { - deferred.reject(error); - }); - - return deferred.promise; - }, -}); - -// Alias built-in field types to public API. -for (let property of MetricsStorageSqliteBackend.prototype._BUILTIN_TYPES) { - this.MetricsStorageBackend[property] = MetricsStorageSqliteBackend.prototype[property]; -} - diff --git a/services/metrics/tests/xpcshell/head.js b/services/metrics/tests/xpcshell/head.js deleted file mode 100644 index cbb4d75075..0000000000 --- a/services/metrics/tests/xpcshell/head.js +++ /dev/null @@ -1,15 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -"use strict"; - -(function initMetricsTestingInfrastructure() { - do_get_profile(); - - let ns = {}; - Components.utils.import("resource://testing-common/services/common/logging.js", - ns); - - ns.initTestLogging("Trace"); -}).call(this); - diff --git a/services/metrics/tests/xpcshell/test_load_modules.js b/services/metrics/tests/xpcshell/test_load_modules.js deleted file mode 100644 index d6e1127cd8..0000000000 --- a/services/metrics/tests/xpcshell/test_load_modules.js +++ /dev/null @@ -1,31 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -"use strict"; - -const modules = [ - "dataprovider.jsm", - "providermanager.jsm", - "storage.jsm", -]; - -const test_modules = [ - "mocks.jsm", -]; - -function run_test() { - for (let m of modules) { - let resource = "resource://gre/modules/services/metrics/" + m; - Components.utils.import(resource, {}); - } - - Components.utils.import("resource://gre/modules/Metrics.jsm", {}); - - for (let m of test_modules) { - let resource = "resource://testing-common/services/metrics/" + m; - Components.utils.import(resource, {}); - } - - Components.utils.import("resource://gre/modules/Metrics.jsm", {}); -} - diff --git a/services/metrics/tests/xpcshell/test_metrics_provider.js b/services/metrics/tests/xpcshell/test_metrics_provider.js deleted file mode 100644 index 2ee6631824..0000000000 --- a/services/metrics/tests/xpcshell/test_metrics_provider.js +++ /dev/null @@ -1,297 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -"use strict"; - -var {utils: Cu} = Components; - -Cu.import("resource://gre/modules/Metrics.jsm"); -Cu.import("resource://gre/modules/Preferences.jsm"); -Cu.import("resource://gre/modules/Task.jsm"); -Cu.import("resource://testing-common/services/metrics/mocks.jsm"); - - -const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000; - - -function getProvider(storageName) { - return Task.spawn(function () { - let provider = new DummyProvider(); - let storage = yield Metrics.Storage(storageName); - - yield provider.init(storage); - - throw new Task.Result(provider); - }); -} - - -function run_test() { - run_next_test(); -}; - -add_test(function test_constructor() { - let failed = false; - try { - new Metrics.Provider(); - } catch(ex) { - do_check_true(ex.message.startsWith("Provider must define a name")); - failed = true; - } - finally { - do_check_true(failed); - } - - run_next_test(); -}); - -add_task(function test_init() { - let provider = new DummyProvider(); - let storage = yield Metrics.Storage("init"); - - yield provider.init(storage); - - let m = provider.getMeasurement("DummyMeasurement", 1); - do_check_true(m instanceof Metrics.Measurement); - do_check_eq(m.id, 1); - do_check_eq(Object.keys(m._fields).length, 7); - do_check_true(m.hasField("daily-counter")); - do_check_false(m.hasField("does-not-exist")); - - yield storage.close(); -}); - -add_task(function test_default_collectors() { - let provider = new DummyProvider(); - let storage = yield Metrics.Storage("default_collectors"); - yield provider.init(storage); - - for (let property in Metrics.Provider.prototype) { - if (!property.startsWith("collect")) { - continue; - } - - let result = provider[property](); - do_check_neq(result, null); - do_check_eq(typeof(result.then), "function"); - } - - yield storage.close(); -}); - -add_task(function test_measurement_storage_basic() { - let provider = yield getProvider("measurement_storage_basic"); - let m = provider.getMeasurement("DummyMeasurement", 1); - - let now = new Date(); - let yesterday = new Date(now.getTime() - MILLISECONDS_PER_DAY); - - // Daily counter. - let counterID = m.fieldID("daily-counter"); - yield m.incrementDailyCounter("daily-counter", now); - yield m.incrementDailyCounter("daily-counter", now); - yield m.incrementDailyCounter("daily-counter", yesterday); - let count = yield provider.storage.getDailyCounterCountFromFieldID(counterID, now); - do_check_eq(count, 2); - - count = yield provider.storage.getDailyCounterCountFromFieldID(counterID, yesterday); - do_check_eq(count, 1); - - yield m.incrementDailyCounter("daily-counter", now, 4); - count = yield provider.storage.getDailyCounterCountFromFieldID(counterID, now); - do_check_eq(count, 6); - - // Daily discrete numeric. - let dailyDiscreteNumericID = m.fieldID("daily-discrete-numeric"); - yield m.addDailyDiscreteNumeric("daily-discrete-numeric", 5, now); - yield m.addDailyDiscreteNumeric("daily-discrete-numeric", 6, now); - yield m.addDailyDiscreteNumeric("daily-discrete-numeric", 7, yesterday); - - let values = yield provider.storage.getDailyDiscreteNumericFromFieldID( - dailyDiscreteNumericID, now); - - do_check_eq(values.size, 1); - do_check_true(values.hasDay(now)); - let actual = values.getDay(now); - do_check_eq(actual.length, 2); - do_check_eq(actual[0], 5); - do_check_eq(actual[1], 6); - - values = yield provider.storage.getDailyDiscreteNumericFromFieldID( - dailyDiscreteNumericID, yesterday); - - do_check_eq(values.size, 1); - do_check_true(values.hasDay(yesterday)); - do_check_eq(values.getDay(yesterday)[0], 7); - - // Daily discrete text. - let dailyDiscreteTextID = m.fieldID("daily-discrete-text"); - yield m.addDailyDiscreteText("daily-discrete-text", "foo", now); - yield m.addDailyDiscreteText("daily-discrete-text", "bar", now); - yield m.addDailyDiscreteText("daily-discrete-text", "biz", yesterday); - - values = yield provider.storage.getDailyDiscreteTextFromFieldID( - dailyDiscreteTextID, now); - - do_check_eq(values.size, 1); - do_check_true(values.hasDay(now)); - actual = values.getDay(now); - do_check_eq(actual.length, 2); - do_check_eq(actual[0], "foo"); - do_check_eq(actual[1], "bar"); - - values = yield provider.storage.getDailyDiscreteTextFromFieldID( - dailyDiscreteTextID, yesterday); - do_check_true(values.hasDay(yesterday)); - do_check_eq(values.getDay(yesterday)[0], "biz"); - - // Daily last numeric. - let lastDailyNumericID = m.fieldID("daily-last-numeric"); - yield m.setDailyLastNumeric("daily-last-numeric", 5, now); - yield m.setDailyLastNumeric("daily-last-numeric", 6, yesterday); - - let result = yield provider.storage.getDailyLastNumericFromFieldID( - lastDailyNumericID, now); - do_check_eq(result.size, 1); - do_check_true(result.hasDay(now)); - do_check_eq(result.getDay(now), 5); - - result = yield provider.storage.getDailyLastNumericFromFieldID( - lastDailyNumericID, yesterday); - do_check_true(result.hasDay(yesterday)); - do_check_eq(result.getDay(yesterday), 6); - - yield m.setDailyLastNumeric("daily-last-numeric", 7, now); - result = yield provider.storage.getDailyLastNumericFromFieldID( - lastDailyNumericID, now); - do_check_eq(result.getDay(now), 7); - - // Daily last text. - let lastDailyTextID = m.fieldID("daily-last-text"); - yield m.setDailyLastText("daily-last-text", "foo", now); - yield m.setDailyLastText("daily-last-text", "bar", yesterday); - - result = yield provider.storage.getDailyLastTextFromFieldID( - lastDailyTextID, now); - do_check_eq(result.size, 1); - do_check_true(result.hasDay(now)); - do_check_eq(result.getDay(now), "foo"); - - result = yield provider.storage.getDailyLastTextFromFieldID( - lastDailyTextID, yesterday); - do_check_true(result.hasDay(yesterday)); - do_check_eq(result.getDay(yesterday), "bar"); - - yield m.setDailyLastText("daily-last-text", "biz", now); - result = yield provider.storage.getDailyLastTextFromFieldID( - lastDailyTextID, now); - do_check_eq(result.getDay(now), "biz"); - - // Last numeric. - let lastNumericID = m.fieldID("last-numeric"); - yield m.setLastNumeric("last-numeric", 1, now); - result = yield provider.storage.getLastNumericFromFieldID(lastNumericID); - do_check_eq(result[1], 1); - do_check_true(result[0].getTime() < now.getTime()); - do_check_true(result[0].getTime() > yesterday.getTime()); - - yield m.setLastNumeric("last-numeric", 2, now); - result = yield provider.storage.getLastNumericFromFieldID(lastNumericID); - do_check_eq(result[1], 2); - - // Last text. - let lastTextID = m.fieldID("last-text"); - yield m.setLastText("last-text", "foo", now); - result = yield provider.storage.getLastTextFromFieldID(lastTextID); - do_check_eq(result[1], "foo"); - do_check_true(result[0].getTime() < now.getTime()); - do_check_true(result[0].getTime() > yesterday.getTime()); - - yield m.setLastText("last-text", "bar", now); - result = yield provider.storage.getLastTextFromFieldID(lastTextID); - do_check_eq(result[1], "bar"); - - yield provider.storage.close(); -}); - -add_task(function test_serialize_json_default() { - let provider = yield getProvider("serialize_json_default"); - - let now = new Date(); - let yesterday = new Date(now.getTime() - MILLISECONDS_PER_DAY); - - let m = provider.getMeasurement("DummyMeasurement", 1); - - m.incrementDailyCounter("daily-counter", now); - m.incrementDailyCounter("daily-counter", now); - m.incrementDailyCounter("daily-counter", yesterday); - - m.addDailyDiscreteNumeric("daily-discrete-numeric", 1, now); - m.addDailyDiscreteNumeric("daily-discrete-numeric", 2, now); - m.addDailyDiscreteNumeric("daily-discrete-numeric", 3, yesterday); - - m.addDailyDiscreteText("daily-discrete-text", "foo", now); - m.addDailyDiscreteText("daily-discrete-text", "bar", now); - m.addDailyDiscreteText("daily-discrete-text", "baz", yesterday); - - m.setDailyLastNumeric("daily-last-numeric", 4, now); - m.setDailyLastNumeric("daily-last-numeric", 5, yesterday); - - m.setDailyLastText("daily-last-text", "apple", now); - m.setDailyLastText("daily-last-text", "orange", yesterday); - - m.setLastNumeric("last-numeric", 6, now); - yield m.setLastText("last-text", "hello", now); - - let data = yield provider.storage.getMeasurementValues(m.id); - - let serializer = m.serializer(m.SERIALIZE_JSON); - let formatted = serializer.singular(data.singular); - - do_check_eq(Object.keys(formatted).length, 3); // Our keys + _v. - do_check_true("last-numeric" in formatted); - do_check_true("last-text" in formatted); - do_check_eq(formatted["last-numeric"], 6); - do_check_eq(formatted["last-text"], "hello"); - do_check_eq(formatted["_v"], 1); - - formatted = serializer.daily(data.days.getDay(now)); - do_check_eq(Object.keys(formatted).length, 6); // Our keys + _v. - do_check_eq(formatted["daily-counter"], 2); - do_check_eq(formatted["_v"], 1); - - do_check_true(Array.isArray(formatted["daily-discrete-numeric"])); - do_check_eq(formatted["daily-discrete-numeric"].length, 2); - do_check_eq(formatted["daily-discrete-numeric"][0], 1); - do_check_eq(formatted["daily-discrete-numeric"][1], 2); - - do_check_true(Array.isArray(formatted["daily-discrete-text"])); - do_check_eq(formatted["daily-discrete-text"].length, 2); - do_check_eq(formatted["daily-discrete-text"][0], "foo"); - do_check_eq(formatted["daily-discrete-text"][1], "bar"); - - do_check_eq(formatted["daily-last-numeric"], 4); - do_check_eq(formatted["daily-last-text"], "apple"); - - formatted = serializer.daily(data.days.getDay(yesterday)); - do_check_eq(formatted["daily-last-numeric"], 5); - do_check_eq(formatted["daily-last-text"], "orange"); - - // Now let's turn off a field so that it's present in the DB - // but not present in the output. - let called = false; - let excluded = "daily-last-numeric"; - Object.defineProperty(m, "shouldIncludeField", { - value: function fakeShouldIncludeField(field) { - called = true; - return field != excluded; - }, - }); - - let limited = serializer.daily(data.days.getDay(yesterday)); - do_check_true(called); - do_check_false(excluded in limited); - do_check_eq(formatted["daily-last-text"], "orange"); - - yield provider.storage.close(); -}); diff --git a/services/metrics/tests/xpcshell/test_metrics_provider_manager.js b/services/metrics/tests/xpcshell/test_metrics_provider_manager.js deleted file mode 100644 index bad8e67bcc..0000000000 --- a/services/metrics/tests/xpcshell/test_metrics_provider_manager.js +++ /dev/null @@ -1,357 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -"use strict"; - -var {classes: Cc, interfaces: Ci, utils: Cu} = Components; - -Cu.import("resource://gre/modules/Promise.jsm"); -Cu.import("resource://gre/modules/Metrics.jsm"); -Cu.import("resource://testing-common/services/metrics/mocks.jsm"); - -const PULL_ONLY_TESTING_CATEGORY = "testing-only-pull-only-providers"; - -function run_test() { - let cm = Cc["@mozilla.org/categorymanager;1"] - .getService(Ci.nsICategoryManager); - cm.addCategoryEntry(PULL_ONLY_TESTING_CATEGORY, "DummyProvider", - "resource://testing-common/services/metrics/mocks.jsm", - false, true); - cm.addCategoryEntry(PULL_ONLY_TESTING_CATEGORY, "DummyConstantProvider", - "resource://testing-common/services/metrics/mocks.jsm", - false, true); - - run_next_test(); -}; - -add_task(function test_constructor() { - let storage = yield Metrics.Storage("constructor"); - let manager = new Metrics.ProviderManager(storage); - - yield storage.close(); -}); - -add_task(function test_register_provider() { - let storage = yield Metrics.Storage("register_provider"); - - let manager = new Metrics.ProviderManager(storage); - let dummy = new DummyProvider(); - - yield manager.registerProvider(dummy); - do_check_eq(manager._providers.size, 1); - yield manager.registerProvider(dummy); - do_check_eq(manager._providers.size, 1); - do_check_eq(manager.getProvider(dummy.name), dummy); - - let failed = false; - try { - manager.registerProvider({}); - } catch (ex) { - do_check_true(ex.message.startsWith("Provider is not valid")); - failed = true; - } finally { - do_check_true(failed); - failed = false; - } - - manager.unregisterProvider(dummy.name); - do_check_eq(manager._providers.size, 0); - do_check_null(manager.getProvider(dummy.name)); - - yield storage.close(); -}); - -add_task(function test_register_providers_from_category_manager() { - const category = "metrics-providers-js-modules"; - - let cm = Cc["@mozilla.org/categorymanager;1"] - .getService(Ci.nsICategoryManager); - cm.addCategoryEntry(category, "DummyProvider", - "resource://testing-common/services/metrics/mocks.jsm", - false, true); - - let storage = yield Metrics.Storage("register_providers_from_category_manager"); - let manager = new Metrics.ProviderManager(storage); - try { - do_check_eq(manager._providers.size, 0); - yield manager.registerProvidersFromCategoryManager(category); - do_check_eq(manager._providers.size, 1); - } finally { - yield storage.close(); - } -}); - -add_task(function test_collect_constant_data() { - let storage = yield Metrics.Storage("collect_constant_data"); - let errorCount = 0; - let manager= new Metrics.ProviderManager(storage); - manager.onProviderError = function () { errorCount++; } - let provider = new DummyProvider(); - yield manager.registerProvider(provider); - - do_check_eq(provider.collectConstantCount, 0); - - yield manager.collectConstantData(); - do_check_eq(provider.collectConstantCount, 1); - - do_check_true(manager._providers.get("DummyProvider").constantsCollected); - - yield storage.close(); - do_check_eq(errorCount, 0); -}); - -add_task(function test_collect_constant_throws() { - let storage = yield Metrics.Storage("collect_constant_throws"); - let manager = new Metrics.ProviderManager(storage); - let errors = []; - manager.onProviderError = function (error) { errors.push(error); }; - - let provider = new DummyProvider(); - provider.throwDuringCollectConstantData = "Fake error during collect"; - yield manager.registerProvider(provider); - - yield manager.collectConstantData(); - do_check_eq(errors.length, 1); - do_check_true(errors[0].includes(provider.throwDuringCollectConstantData)); - - yield storage.close(); -}); - -add_task(function test_collect_constant_populate_throws() { - let storage = yield Metrics.Storage("collect_constant_populate_throws"); - let manager = new Metrics.ProviderManager(storage); - let errors = []; - manager.onProviderError = function (error) { errors.push(error); }; - - let provider = new DummyProvider(); - provider.throwDuringConstantPopulate = "Fake error during constant populate"; - yield manager.registerProvider(provider); - - yield manager.collectConstantData(); - - do_check_eq(errors.length, 1); - do_check_true(errors[0].includes(provider.throwDuringConstantPopulate)); - do_check_false(manager._providers.get(provider.name).constantsCollected); - - yield storage.close(); -}); - -add_task(function test_collect_constant_onetime() { - let storage = yield Metrics.Storage("collect_constant_onetime"); - let manager = new Metrics.ProviderManager(storage); - let provider = new DummyProvider(); - yield manager.registerProvider(provider); - - yield manager.collectConstantData(); - do_check_eq(provider.collectConstantCount, 1); - - yield manager.collectConstantData(); - do_check_eq(provider.collectConstantCount, 1); - - yield storage.close(); -}); - -add_task(function test_collect_multiple() { - let storage = yield Metrics.Storage("collect_multiple"); - let manager = new Metrics.ProviderManager(storage); - - for (let i = 0; i < 10; i++) { - yield manager.registerProvider(new DummyProvider("provider" + i)); - } - - do_check_eq(manager._providers.size, 10); - - yield manager.collectConstantData(); - - yield storage.close(); -}); - -add_task(function test_collect_daily() { - let storage = yield Metrics.Storage("collect_daily"); - let manager = new Metrics.ProviderManager(storage); - - let provider1 = new DummyProvider("DP1"); - let provider2 = new DummyProvider("DP2"); - - yield manager.registerProvider(provider1); - yield manager.registerProvider(provider2); - - yield manager.collectDailyData(); - do_check_eq(provider1.collectDailyCount, 1); - do_check_eq(provider2.collectDailyCount, 1); - - yield manager.collectDailyData(); - do_check_eq(provider1.collectDailyCount, 2); - do_check_eq(provider2.collectDailyCount, 2); - - yield storage.close(); -}); - -add_task(function test_pull_only_not_initialized() { - let storage = yield Metrics.Storage("pull_only_not_initialized"); - let manager = new Metrics.ProviderManager(storage); - yield manager.registerProvidersFromCategoryManager(PULL_ONLY_TESTING_CATEGORY); - do_check_eq(manager.providers.length, 1); - do_check_eq(manager.providers[0].name, "DummyProvider"); - yield storage.close(); -}); - -add_task(function test_pull_only_registration() { - let storage = yield Metrics.Storage("pull_only_registration"); - let manager = new Metrics.ProviderManager(storage); - yield manager.registerProvidersFromCategoryManager(PULL_ONLY_TESTING_CATEGORY); - do_check_eq(manager.providers.length, 1); - - // Simple registration and unregistration. - yield manager.ensurePullOnlyProvidersRegistered(); - do_check_eq(manager.providers.length, 2); - do_check_neq(manager.getProvider("DummyConstantProvider"), null); - yield manager.ensurePullOnlyProvidersUnregistered(); - do_check_eq(manager.providers.length, 1); - do_check_null(manager.getProvider("DummyConstantProvider")); - - // Multiple calls to register work. - yield manager.ensurePullOnlyProvidersRegistered(); - do_check_eq(manager.providers.length, 2); - yield manager.ensurePullOnlyProvidersRegistered(); - do_check_eq(manager.providers.length, 2); - - // Unregister with 2 requests for registration should not unregister. - yield manager.ensurePullOnlyProvidersUnregistered(); - do_check_eq(manager.providers.length, 2); - - // But the 2nd one will. - yield manager.ensurePullOnlyProvidersUnregistered(); - do_check_eq(manager.providers.length, 1); - - yield storage.close(); -}); - -add_task(function test_pull_only_register_while_registering() { - let storage = yield Metrics.Storage("pull_only_register_will_registering"); - let manager = new Metrics.ProviderManager(storage); - yield manager.registerProvidersFromCategoryManager(PULL_ONLY_TESTING_CATEGORY); - - manager.ensurePullOnlyProvidersRegistered(); - manager.ensurePullOnlyProvidersRegistered(); - yield manager.ensurePullOnlyProvidersRegistered(); - do_check_eq(manager.providers.length, 2); - - manager.ensurePullOnlyProvidersUnregistered(); - manager.ensurePullOnlyProvidersUnregistered(); - yield manager.ensurePullOnlyProvidersUnregistered(); - do_check_eq(manager.providers.length, 1); - - yield storage.close(); -}); - -add_task(function test_pull_only_unregister_while_registering() { - let storage = yield Metrics.Storage("pull_only_unregister_while_registering"); - let manager = new Metrics.ProviderManager(storage); - yield manager.registerProvidersFromCategoryManager(PULL_ONLY_TESTING_CATEGORY); - - manager.ensurePullOnlyProvidersRegistered(); - yield manager.ensurePullOnlyProvidersUnregistered(); - do_check_eq(manager.providers.length, 1); - - yield storage.close(); -}); - -add_task(function test_pull_only_register_while_unregistering() { - let storage = yield Metrics.Storage("pull_only_register_while_unregistering"); - let manager = new Metrics.ProviderManager(storage); - yield manager.registerProvidersFromCategoryManager(PULL_ONLY_TESTING_CATEGORY); - - yield manager.ensurePullOnlyProvidersRegistered(); - manager.ensurePullOnlyProvidersUnregistered(); - yield manager.ensurePullOnlyProvidersRegistered(); - do_check_eq(manager.providers.length, 2); - - yield storage.close(); -}); - -// Re-use database for perf reasons. -const REGISTRATION_ERRORS_DB = "registration_errors"; - -add_task(function test_category_manager_registration_error() { - let storage = yield Metrics.Storage(REGISTRATION_ERRORS_DB); - let manager = new Metrics.ProviderManager(storage); - - let cm = Cc["@mozilla.org/categorymanager;1"] - .getService(Ci.nsICategoryManager); - cm.addCategoryEntry("registration-errors", "DummyThrowOnInitProvider", - "resource://testing-common/services/metrics/mocks.jsm", - false, true); - - let deferred = Promise.defer(); - let errorCount = 0; - - manager.onProviderError = function (msg) { - errorCount++; - deferred.resolve(msg); - }; - - yield manager.registerProvidersFromCategoryManager("registration-errors"); - do_check_eq(manager.providers.length, 0); - do_check_eq(errorCount, 1); - - let msg = yield deferred.promise; - do_check_true(msg.includes("Provider error: DummyThrowOnInitProvider: " - + "Error registering provider from category manager: " - + "Error: Dummy Error")); - - yield storage.close(); -}); - -add_task(function test_pull_only_registration_error() { - let storage = yield Metrics.Storage(REGISTRATION_ERRORS_DB); - let manager = new Metrics.ProviderManager(storage); - - let deferred = Promise.defer(); - let errorCount = 0; - - manager.onProviderError = function (msg) { - errorCount++; - deferred.resolve(msg); - }; - - yield manager.registerProviderFromType(DummyPullOnlyThrowsOnInitProvider); - do_check_eq(errorCount, 0); - - yield manager.ensurePullOnlyProvidersRegistered(); - do_check_eq(errorCount, 1); - - let msg = yield deferred.promise; - do_check_true(msg.includes("Provider error: DummyPullOnlyThrowsOnInitProvider: " + - "Error registering pull-only provider: Error: Dummy Error")); - - yield storage.close(); -}); - -add_task(function test_error_during_shutdown() { - let storage = yield Metrics.Storage(REGISTRATION_ERRORS_DB); - let manager = new Metrics.ProviderManager(storage); - - let deferred = Promise.defer(); - let errorCount = 0; - - manager.onProviderError = function (msg) { - errorCount++; - deferred.resolve(msg); - }; - - yield manager.registerProviderFromType(DummyThrowOnShutdownProvider); - yield manager.registerProviderFromType(DummyProvider); - do_check_eq(errorCount, 0); - do_check_eq(manager.providers.length, 1); - - yield manager.ensurePullOnlyProvidersRegistered(); - do_check_eq(errorCount, 0); - yield manager.ensurePullOnlyProvidersUnregistered(); - do_check_eq(errorCount, 1); - let msg = yield deferred.promise; - do_check_true(msg.includes("Provider error: DummyThrowOnShutdownProvider: " + - "Error when shutting down provider: Error: Dummy shutdown error")); - - yield storage.close(); -}); diff --git a/services/metrics/tests/xpcshell/test_metrics_storage.js b/services/metrics/tests/xpcshell/test_metrics_storage.js deleted file mode 100644 index 2ee2ad6d2c..0000000000 --- a/services/metrics/tests/xpcshell/test_metrics_storage.js +++ /dev/null @@ -1,839 +0,0 @@ -/* Any copyright is dedicated to the Public Domain. - * http://creativecommons.org/publicdomain/zero/1.0/ */ - -"use strict"; - -var {utils: Cu} = Components; - -Cu.import("resource://gre/modules/Promise.jsm"); -Cu.import("resource://gre/modules/Metrics.jsm"); -Cu.import("resource://services-common/utils.js"); - - -const MILLISECONDS_PER_DAY = 24 * 60 * 60 * 1000; - - -function run_test() { - run_next_test(); -} - -add_test(function test_days_date_conversion() { - let toDays = Metrics.dateToDays; - let toDate = Metrics.daysToDate; - - let d = new Date(0); - do_check_eq(toDays(d), 0); - - d = new Date(MILLISECONDS_PER_DAY); - do_check_eq(toDays(d), 1); - - d = new Date(MILLISECONDS_PER_DAY - 1); - do_check_eq(toDays(d), 0); - - d = new Date("1970-12-31T23:59:59.999Z"); - do_check_eq(toDays(d), 364); - - d = new Date("1971-01-01T00:00:00Z"); - do_check_eq(toDays(d), 365); - - d = toDate(0); - do_check_eq(d.getTime(), 0); - - d = toDate(1); - do_check_eq(d.getTime(), MILLISECONDS_PER_DAY); - - d = toDate(365); - do_check_eq(d.getUTCFullYear(), 1971); - do_check_eq(d.getUTCMonth(), 0); - do_check_eq(d.getUTCDate(), 1); - do_check_eq(d.getUTCHours(), 0); - do_check_eq(d.getUTCMinutes(), 0); - do_check_eq(d.getUTCSeconds(), 0); - do_check_eq(d.getUTCMilliseconds(), 0); - - run_next_test(); -}); - -add_task(function test_get_sqlite_backend() { - let backend = yield Metrics.Storage("get_sqlite_backend.sqlite"); - - do_check_neq(backend._connection, null); - - // Ensure WAL and auto checkpoint are enabled. - do_check_neq(backend._enabledWALCheckpointPages, null); - let rows = yield backend._connection.execute("PRAGMA journal_mode"); - do_check_eq(rows[0].getResultByIndex(0), "wal"); - rows = yield backend._connection.execute("PRAGMA wal_autocheckpoint"); - do_check_eq(rows[0].getResultByIndex(0), backend._enabledWALCheckpointPages); - - yield backend.close(); - do_check_null(backend._connection); -}); - -add_task(function test_reconnect() { - let backend = yield Metrics.Storage("reconnect"); - yield backend.close(); - - let backend2 = yield Metrics.Storage("reconnect"); - yield backend2.close(); -}); - -add_task(function test_future_schema_errors() { - let backend = yield Metrics.Storage("future_schema_errors"); - yield backend._connection.setSchemaVersion(2); - yield backend.close(); - - let backend2; - let failed = false; - try { - backend2 = yield Metrics.Storage("future_schema_errors"); - } catch (ex) { - failed = true; - do_check_true(ex.message.startsWith("Unknown database schema")); - } - - do_check_null(backend2); - do_check_true(failed); -}); - -add_task(function test_checkpoint_apis() { - let backend = yield Metrics.Storage("checkpoint_apis"); - let c = backend._connection; - let count = c._connectionData._statementCounter; - - yield backend.setAutoCheckpoint(0); - do_check_eq(c._connectionData._statementCounter, count + 1); - - let rows = yield c.execute("PRAGMA wal_autocheckpoint"); - do_check_eq(rows[0].getResultByIndex(0), 0); - count = c._connectionData._statementCounter; - - yield backend.setAutoCheckpoint(1); - do_check_eq(c._connectionData._statementCounter, count + 1); - - rows = yield c.execute("PRAGMA wal_autocheckpoint"); - do_check_eq(rows[0].getResultByIndex(0), backend._enabledWALCheckpointPages); - count = c._connectionData._statementCounter; - - yield backend.checkpoint(); - do_check_eq(c._connectionData._statementCounter, count + 1); - - yield backend.checkpoint(); - do_check_eq(c._connectionData._statementCounter, count + 2); - - yield backend.close(); -}); - -add_task(function test_measurement_registration() { - let backend = yield Metrics.Storage("measurement_registration"); - - do_check_false(backend.hasProvider("foo")); - do_check_false(backend.hasMeasurement("foo", "bar", 1)); - - let id = yield backend.registerMeasurement("foo", "bar", 1); - do_check_eq(id, 1); - - do_check_true(backend.hasProvider("foo")); - do_check_true(backend.hasMeasurement("foo", "bar", 1)); - do_check_eq(backend.measurementID("foo", "bar", 1), id); - do_check_false(backend.hasMeasurement("foo", "bar", 2)); - - let id2 = yield backend.registerMeasurement("foo", "bar", 2); - do_check_eq(id2, 2); - do_check_true(backend.hasMeasurement("foo", "bar", 2)); - do_check_eq(backend.measurementID("foo", "bar", 2), id2); - - yield backend.close(); -}); - -add_task(function test_field_registration_basic() { - let backend = yield Metrics.Storage("field_registration_basic"); - - do_check_false(backend.hasField("foo", "bar", 1, "baz")); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - do_check_false(backend.hasField("foo", "bar", 1, "baz")); - do_check_false(backend.hasFieldFromMeasurement(mID, "baz")); - - let bazID = yield backend.registerField(mID, "baz", - backend.FIELD_DAILY_COUNTER); - do_check_true(backend.hasField("foo", "bar", 1, "baz")); - do_check_true(backend.hasFieldFromMeasurement(mID, "baz")); - - let bar2ID = yield backend.registerMeasurement("foo", "bar2", 1); - - yield backend.registerField(bar2ID, "baz", - backend.FIELD_DAILY_DISCRETE_NUMERIC); - - do_check_true(backend.hasField("foo", "bar2", 1, "baz")); - - yield backend.close(); -}); - -// Ensure changes types of fields results in fatal error. -add_task(function test_field_registration_changed_type() { - let backend = yield Metrics.Storage("field_registration_changed_type"); - - let mID = yield backend.registerMeasurement("bar", "bar", 1); - - let id = yield backend.registerField(mID, "baz", - backend.FIELD_DAILY_COUNTER); - - let caught = false; - try { - yield backend.registerField(mID, "baz", - backend.FIELD_DAILY_DISCRETE_NUMERIC); - } catch (ex) { - caught = true; - do_check_true(ex.message.startsWith("Field already defined with different type")); - } - - do_check_true(caught); - - yield backend.close(); -}); - -add_task(function test_field_registration_repopulation() { - let backend = yield Metrics.Storage("field_registration_repopulation"); - - let mID1 = yield backend.registerMeasurement("foo", "bar", 1); - let mID2 = yield backend.registerMeasurement("foo", "bar", 2); - let mID3 = yield backend.registerMeasurement("foo", "biz", 1); - let mID4 = yield backend.registerMeasurement("baz", "foo", 1); - - let fID1 = yield backend.registerField(mID1, "foo", backend.FIELD_DAILY_COUNTER); - let fID2 = yield backend.registerField(mID1, "bar", backend.FIELD_DAILY_DISCRETE_NUMERIC); - let fID3 = yield backend.registerField(mID4, "foo", backend.FIELD_LAST_TEXT); - - yield backend.close(); - - backend = yield Metrics.Storage("field_registration_repopulation"); - - do_check_true(backend.hasProvider("foo")); - do_check_true(backend.hasProvider("baz")); - do_check_true(backend.hasMeasurement("foo", "bar", 1)); - do_check_eq(backend.measurementID("foo", "bar", 1), mID1); - do_check_true(backend.hasMeasurement("foo", "bar", 2)); - do_check_eq(backend.measurementID("foo", "bar", 2), mID2); - do_check_true(backend.hasMeasurement("foo", "biz", 1)); - do_check_eq(backend.measurementID("foo", "biz", 1), mID3); - do_check_true(backend.hasMeasurement("baz", "foo", 1)); - do_check_eq(backend.measurementID("baz", "foo", 1), mID4); - - do_check_true(backend.hasField("foo", "bar", 1, "foo")); - do_check_eq(backend.fieldID("foo", "bar", 1, "foo"), fID1); - do_check_true(backend.hasField("foo", "bar", 1, "bar")); - do_check_eq(backend.fieldID("foo", "bar", 1, "bar"), fID2); - do_check_true(backend.hasField("baz", "foo", 1, "foo")); - do_check_eq(backend.fieldID("baz", "foo", 1, "foo"), fID3); - - yield backend.close(); -}); - -add_task(function test_enqueue_operation_execution_order() { - let backend = yield Metrics.Storage("enqueue_operation_execution_order"); - - let executionCount = 0; - - let fns = { - op1: function () { - do_check_eq(executionCount, 1); - }, - - op2: function () { - do_check_eq(executionCount, 2); - }, - - op3: function () { - do_check_eq(executionCount, 3); - }, - }; - - function enqueuedOperation(fn) { - let deferred = Promise.defer(); - - CommonUtils.nextTick(function onNextTick() { - executionCount++; - fn(); - deferred.resolve(); - }); - - return deferred.promise; - } - - let promises = []; - for (let i = 1; i <= 3; i++) { - let fn = fns["op" + i]; - promises.push(backend.enqueueOperation(enqueuedOperation.bind(this, fn))); - } - - for (let promise of promises) { - yield promise; - } - - yield backend.close(); -}); - -add_task(function test_enqueue_operation_many() { - let backend = yield Metrics.Storage("enqueue_operation_many"); - - let promises = []; - for (let i = 0; i < 100; i++) { - promises.push(backend.registerMeasurement("foo", "bar" + i, 1)); - } - - for (let promise of promises) { - yield promise; - } - - yield backend.close(); -}); - -// If the operation did not return a promise, everything should still execute. -add_task(function test_enqueue_operation_no_return_promise() { - let backend = yield Metrics.Storage("enqueue_operation_no_return_promise"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let fID = yield backend.registerField(mID, "baz", backend.FIELD_DAILY_COUNTER); - let now = new Date(); - - let promises = []; - for (let i = 0; i < 10; i++) { - promises.push(backend.enqueueOperation(function op() { - backend.incrementDailyCounterFromFieldID(fID, now); - })); - } - - let deferred = Promise.defer(); - - let finished = 0; - for (let promise of promises) { - promise.then( - do_throw.bind(this, "Unexpected resolve."), - function onError() { - finished++; - - if (finished == promises.length) { - backend.getDailyCounterCountFromFieldID(fID, now).then(function onCount(count) { - // There should not be a race condition here because storage - // serializes all statements. So, for the getDailyCounterCount - // query to finish means that all counter update statements must - // have completed. - do_check_eq(count, promises.length); - deferred.resolve(); - }); - } - } - ); - } - - yield deferred.promise; - yield backend.close(); -}); - -// If an operation throws, subsequent operations should still execute. -add_task(function test_enqueue_operation_throw_exception() { - let backend = yield Metrics.Storage("enqueue_operation_rejected_promise"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let fID = yield backend.registerField(mID, "baz", backend.FIELD_DAILY_COUNTER); - let now = new Date(); - - let deferred = Promise.defer(); - backend.enqueueOperation(function bad() { - throw new Error("I failed."); - }).then(do_throw, function onError(error) { - do_check_true(error.message.includes("I failed.")); - deferred.resolve(); - }); - - let promise = backend.enqueueOperation(function () { - return backend.incrementDailyCounterFromFieldID(fID, now); - }); - - yield deferred.promise; - yield promise; - - let count = yield backend.getDailyCounterCountFromFieldID(fID, now); - do_check_eq(count, 1); - yield backend.close(); -}); - -// If an operation rejects, subsequent operations should still execute. -add_task(function test_enqueue_operation_reject_promise() { - let backend = yield Metrics.Storage("enqueue_operation_reject_promise"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let fID = yield backend.registerField(mID, "baz", backend.FIELD_DAILY_COUNTER); - let now = new Date(); - - let deferred = Promise.defer(); - backend.enqueueOperation(function reject() { - let d = Promise.defer(); - - CommonUtils.nextTick(function nextTick() { - d.reject("I failed."); - }); - - return d.promise; - }).then(do_throw, function onError(error) { - deferred.resolve(); - }); - - let promise = backend.enqueueOperation(function () { - return backend.incrementDailyCounterFromFieldID(fID, now); - }); - - yield deferred.promise; - yield promise; - - let count = yield backend.getDailyCounterCountFromFieldID(fID, now); - do_check_eq(count, 1); - yield backend.close(); -}); - -add_task(function test_enqueue_transaction() { - let backend = yield Metrics.Storage("enqueue_transaction"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let fID = yield backend.registerField(mID, "baz", backend.FIELD_DAILY_COUNTER); - let now = new Date(); - - yield backend.incrementDailyCounterFromFieldID(fID, now); - - yield backend.enqueueTransaction(function transaction() { - yield backend.incrementDailyCounterFromFieldID(fID, now); - }); - - let count = yield backend.getDailyCounterCountFromFieldID(fID, now); - do_check_eq(count, 2); - - let errored = false; - try { - yield backend.enqueueTransaction(function aborted() { - yield backend.incrementDailyCounterFromFieldID(fID, now); - - throw new Error("Some error."); - }); - } catch (ex) { - errored = true; - } finally { - do_check_true(errored); - } - - count = yield backend.getDailyCounterCountFromFieldID(fID, now); - do_check_eq(count, 2); - - yield backend.close(); -}); - -add_task(function test_increment_daily_counter_basic() { - let backend = yield Metrics.Storage("increment_daily_counter_basic"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - - let fieldID = yield backend.registerField(mID, "baz", - backend.FIELD_DAILY_COUNTER); - - let now = new Date(); - yield backend.incrementDailyCounterFromFieldID(fieldID, now); - - let count = yield backend.getDailyCounterCountFromFieldID(fieldID, now); - do_check_eq(count, 1); - - yield backend.incrementDailyCounterFromFieldID(fieldID, now); - count = yield backend.getDailyCounterCountFromFieldID(fieldID, now); - do_check_eq(count, 2); - - yield backend.incrementDailyCounterFromFieldID(fieldID, now, 10); - count = yield backend.getDailyCounterCountFromFieldID(fieldID, now); - do_check_eq(count, 12); - - yield backend.close(); -}); - -add_task(function test_increment_daily_counter_multiple_days() { - let backend = yield Metrics.Storage("increment_daily_counter_multiple_days"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let fieldID = yield backend.registerField(mID, "baz", - backend.FIELD_DAILY_COUNTER); - - let days = []; - let now = Date.now(); - for (let i = 0; i < 100; i++) { - days.push(new Date(now - i * MILLISECONDS_PER_DAY)); - } - - for (let day of days) { - yield backend.incrementDailyCounterFromFieldID(fieldID, day); - } - - let result = yield backend.getDailyCounterCountsFromFieldID(fieldID); - do_check_eq(result.size, 100); - for (let day of days) { - do_check_true(result.hasDay(day)); - do_check_eq(result.getDay(day), 1); - } - - let fields = yield backend.getMeasurementDailyCountersFromMeasurementID(mID); - do_check_eq(fields.size, 1); - do_check_true(fields.has("baz")); - do_check_eq(fields.get("baz").size, 100); - - for (let day of days) { - do_check_true(fields.get("baz").hasDay(day)); - do_check_eq(fields.get("baz").getDay(day), 1); - } - - yield backend.close(); -}); - -add_task(function test_last_values() { - let backend = yield Metrics.Storage("set_last"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let numberID = yield backend.registerField(mID, "number", - backend.FIELD_LAST_NUMERIC); - let textID = yield backend.registerField(mID, "text", - backend.FIELD_LAST_TEXT); - let now = new Date(); - let nowDay = new Date(Math.floor(now.getTime() / MILLISECONDS_PER_DAY) * MILLISECONDS_PER_DAY); - - yield backend.setLastNumericFromFieldID(numberID, 42, now); - yield backend.setLastTextFromFieldID(textID, "hello world", now); - - let result = yield backend.getLastNumericFromFieldID(numberID); - do_check_true(Array.isArray(result)); - do_check_eq(result[0].getTime(), nowDay.getTime()); - do_check_eq(typeof(result[1]), "number"); - do_check_eq(result[1], 42); - - result = yield backend.getLastTextFromFieldID(textID); - do_check_true(Array.isArray(result)); - do_check_eq(result[0].getTime(), nowDay.getTime()); - do_check_eq(typeof(result[1]), "string"); - do_check_eq(result[1], "hello world"); - - let missingID = yield backend.registerField(mID, "missing", - backend.FIELD_LAST_NUMERIC); - do_check_null(yield backend.getLastNumericFromFieldID(missingID)); - - let fields = yield backend.getMeasurementLastValuesFromMeasurementID(mID); - do_check_eq(fields.size, 2); - do_check_true(fields.has("number")); - do_check_true(fields.has("text")); - do_check_eq(fields.get("number")[1], 42); - do_check_eq(fields.get("text")[1], "hello world"); - - yield backend.close(); -}); - -add_task(function test_discrete_values_basic() { - let backend = yield Metrics.Storage("discrete_values_basic"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let numericID = yield backend.registerField(mID, "numeric", - backend.FIELD_DAILY_DISCRETE_NUMERIC); - let textID = yield backend.registerField(mID, "text", - backend.FIELD_DAILY_DISCRETE_TEXT); - - let now = new Date(); - let expectedNumeric = []; - let expectedText = []; - for (let i = 0; i < 100; i++) { - expectedNumeric.push(i); - expectedText.push("value" + i); - yield backend.addDailyDiscreteNumericFromFieldID(numericID, i, now); - yield backend.addDailyDiscreteTextFromFieldID(textID, "value" + i, now); - } - - let values = yield backend.getDailyDiscreteNumericFromFieldID(numericID); - do_check_eq(values.size, 1); - do_check_true(values.hasDay(now)); - do_check_true(Array.isArray(values.getDay(now))); - do_check_eq(values.getDay(now).length, expectedNumeric.length); - - for (let i = 0; i < expectedNumeric.length; i++) { - do_check_eq(values.getDay(now)[i], expectedNumeric[i]); - } - - values = yield backend.getDailyDiscreteTextFromFieldID(textID); - do_check_eq(values.size, 1); - do_check_true(values.hasDay(now)); - do_check_true(Array.isArray(values.getDay(now))); - do_check_eq(values.getDay(now).length, expectedText.length); - - for (let i = 0; i < expectedText.length; i++) { - do_check_eq(values.getDay(now)[i], expectedText[i]); - } - - let fields = yield backend.getMeasurementDailyDiscreteValuesFromMeasurementID(mID); - do_check_eq(fields.size, 2); - do_check_true(fields.has("numeric")); - do_check_true(fields.has("text")); - - let numeric = fields.get("numeric"); - let text = fields.get("text"); - do_check_true(numeric.hasDay(now)); - do_check_true(text.hasDay(now)); - do_check_eq(numeric.getDay(now).length, expectedNumeric.length); - do_check_eq(text.getDay(now).length, expectedText.length); - - for (let i = 0; i < expectedNumeric.length; i++) { - do_check_eq(numeric.getDay(now)[i], expectedNumeric[i]); - } - - for (let i = 0; i < expectedText.length; i++) { - do_check_eq(text.getDay(now)[i], expectedText[i]); - } - - yield backend.close(); -}); - -add_task(function test_discrete_values_multiple_days() { - let backend = yield Metrics.Storage("discrete_values_multiple_days"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let id = yield backend.registerField(mID, "baz", - backend.FIELD_DAILY_DISCRETE_NUMERIC); - - let now = new Date(); - let dates = []; - for (let i = 0; i < 50; i++) { - let date = new Date(now.getTime() + i * MILLISECONDS_PER_DAY); - dates.push(date); - - yield backend.addDailyDiscreteNumericFromFieldID(id, i, date); - } - - let values = yield backend.getDailyDiscreteNumericFromFieldID(id); - do_check_eq(values.size, 50); - - let i = 0; - for (let date of dates) { - do_check_true(values.hasDay(date)); - do_check_eq(values.getDay(date)[0], i); - i++; - } - - let fields = yield backend.getMeasurementDailyDiscreteValuesFromMeasurementID(mID); - do_check_eq(fields.size, 1); - do_check_true(fields.has("baz")); - let baz = fields.get("baz"); - do_check_eq(baz.size, 50); - i = 0; - for (let date of dates) { - do_check_true(baz.hasDay(date)); - do_check_eq(baz.getDay(date).length, 1); - do_check_eq(baz.getDay(date)[0], i); - i++; - } - - yield backend.close(); -}); - -add_task(function test_daily_last_values() { - let backend = yield Metrics.Storage("daily_last_values"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let numericID = yield backend.registerField(mID, "numeric", - backend.FIELD_DAILY_LAST_NUMERIC); - let textID = yield backend.registerField(mID, "text", - backend.FIELD_DAILY_LAST_TEXT); - - let now = new Date(); - let yesterday = new Date(now.getTime() - MILLISECONDS_PER_DAY); - let dayBefore = new Date(yesterday.getTime() - MILLISECONDS_PER_DAY); - - yield backend.setDailyLastNumericFromFieldID(numericID, 1, yesterday); - yield backend.setDailyLastNumericFromFieldID(numericID, 2, now); - yield backend.setDailyLastNumericFromFieldID(numericID, 3, dayBefore); - yield backend.setDailyLastTextFromFieldID(textID, "foo", now); - yield backend.setDailyLastTextFromFieldID(textID, "bar", yesterday); - yield backend.setDailyLastTextFromFieldID(textID, "baz", dayBefore); - - let days = yield backend.getDailyLastNumericFromFieldID(numericID); - do_check_eq(days.size, 3); - do_check_eq(days.getDay(yesterday), 1); - do_check_eq(days.getDay(now), 2); - do_check_eq(days.getDay(dayBefore), 3); - - days = yield backend.getDailyLastTextFromFieldID(textID); - do_check_eq(days.size, 3); - do_check_eq(days.getDay(now), "foo"); - do_check_eq(days.getDay(yesterday), "bar"); - do_check_eq(days.getDay(dayBefore), "baz"); - - yield backend.setDailyLastNumericFromFieldID(numericID, 4, yesterday); - days = yield backend.getDailyLastNumericFromFieldID(numericID); - do_check_eq(days.getDay(yesterday), 4); - - yield backend.setDailyLastTextFromFieldID(textID, "biz", yesterday); - days = yield backend.getDailyLastTextFromFieldID(textID); - do_check_eq(days.getDay(yesterday), "biz"); - - days = yield backend.getDailyLastNumericFromFieldID(numericID, yesterday); - do_check_eq(days.size, 1); - do_check_eq(days.getDay(yesterday), 4); - - days = yield backend.getDailyLastTextFromFieldID(textID, yesterday); - do_check_eq(days.size, 1); - do_check_eq(days.getDay(yesterday), "biz"); - - let fields = yield backend.getMeasurementDailyLastValuesFromMeasurementID(mID); - do_check_eq(fields.size, 2); - do_check_true(fields.has("numeric")); - do_check_true(fields.has("text")); - let numeric = fields.get("numeric"); - let text = fields.get("text"); - do_check_true(numeric.hasDay(yesterday)); - do_check_true(numeric.hasDay(dayBefore)); - do_check_true(numeric.hasDay(now)); - do_check_true(text.hasDay(yesterday)); - do_check_true(text.hasDay(dayBefore)); - do_check_true(text.hasDay(now)); - do_check_eq(numeric.getDay(yesterday), 4); - do_check_eq(text.getDay(yesterday), "biz"); - - yield backend.close(); -}); - -add_task(function test_prune_data_before() { - let backend = yield Metrics.Storage("prune_data_before"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - - let counterID = yield backend.registerField(mID, "baz", - backend.FIELD_DAILY_COUNTER); - let text1ID = yield backend.registerField(mID, "one_text_1", - backend.FIELD_LAST_TEXT); - let text2ID = yield backend.registerField(mID, "one_text_2", - backend.FIELD_LAST_TEXT); - let numeric1ID = yield backend.registerField(mID, "one_numeric_1", - backend.FIELD_LAST_NUMERIC); - let numeric2ID = yield backend.registerField(mID, "one_numeric_2", - backend.FIELD_LAST_NUMERIC); - let text3ID = yield backend.registerField(mID, "daily_last_text_1", - backend.FIELD_DAILY_LAST_TEXT); - let text4ID = yield backend.registerField(mID, "daily_last_text_2", - backend.FIELD_DAILY_LAST_TEXT); - let numeric3ID = yield backend.registerField(mID, "daily_last_numeric_1", - backend.FIELD_DAILY_LAST_NUMERIC); - let numeric4ID = yield backend.registerField(mID, "daily_last_numeric_2", - backend.FIELD_DAILY_LAST_NUMERIC); - - let now = new Date(); - let yesterday = new Date(now.getTime() - MILLISECONDS_PER_DAY); - let dayBefore = new Date(yesterday.getTime() - MILLISECONDS_PER_DAY); - - yield backend.incrementDailyCounterFromFieldID(counterID, now); - yield backend.incrementDailyCounterFromFieldID(counterID, yesterday); - yield backend.incrementDailyCounterFromFieldID(counterID, dayBefore); - yield backend.setLastTextFromFieldID(text1ID, "hello", dayBefore); - yield backend.setLastTextFromFieldID(text2ID, "world", yesterday); - yield backend.setLastNumericFromFieldID(numeric1ID, 42, dayBefore); - yield backend.setLastNumericFromFieldID(numeric2ID, 43, yesterday); - yield backend.setDailyLastTextFromFieldID(text3ID, "foo", dayBefore); - yield backend.setDailyLastTextFromFieldID(text3ID, "bar", yesterday); - yield backend.setDailyLastTextFromFieldID(text4ID, "hello", dayBefore); - yield backend.setDailyLastTextFromFieldID(text4ID, "world", yesterday); - yield backend.setDailyLastNumericFromFieldID(numeric3ID, 40, dayBefore); - yield backend.setDailyLastNumericFromFieldID(numeric3ID, 41, yesterday); - yield backend.setDailyLastNumericFromFieldID(numeric4ID, 42, dayBefore); - yield backend.setDailyLastNumericFromFieldID(numeric4ID, 43, yesterday); - - let days = yield backend.getDailyCounterCountsFromFieldID(counterID); - do_check_eq(days.size, 3); - - yield backend.pruneDataBefore(yesterday); - days = yield backend.getDailyCounterCountsFromFieldID(counterID); - do_check_eq(days.size, 2); - do_check_false(days.hasDay(dayBefore)); - - do_check_null(yield backend.getLastTextFromFieldID(text1ID)); - do_check_null(yield backend.getLastNumericFromFieldID(numeric1ID)); - - let result = yield backend.getLastTextFromFieldID(text2ID); - do_check_true(Array.isArray(result)); - do_check_eq(result[1], "world"); - - result = yield backend.getLastNumericFromFieldID(numeric2ID); - do_check_true(Array.isArray(result)); - do_check_eq(result[1], 43); - - result = yield backend.getDailyLastNumericFromFieldID(numeric3ID); - do_check_eq(result.size, 1); - do_check_true(result.hasDay(yesterday)); - - result = yield backend.getDailyLastTextFromFieldID(text3ID); - do_check_eq(result.size, 1); - do_check_true(result.hasDay(yesterday)); - - yield backend.close(); -}); - -add_task(function test_provider_state() { - let backend = yield Metrics.Storage("provider_state"); - - yield backend.registerMeasurement("foo", "bar", 1); - yield backend.setProviderState("foo", "apple", "orange"); - let value = yield backend.getProviderState("foo", "apple"); - do_check_eq(value, "orange"); - - yield backend.setProviderState("foo", "apple", "pear"); - value = yield backend.getProviderState("foo", "apple"); - do_check_eq(value, "pear"); - - yield backend.close(); -}); - -add_task(function test_get_measurement_values() { - let backend = yield Metrics.Storage("get_measurement_values"); - - let mID = yield backend.registerMeasurement("foo", "bar", 1); - let id1 = yield backend.registerField(mID, "id1", backend.FIELD_DAILY_COUNTER); - let id2 = yield backend.registerField(mID, "id2", backend.FIELD_DAILY_DISCRETE_NUMERIC); - let id3 = yield backend.registerField(mID, "id3", backend.FIELD_DAILY_DISCRETE_TEXT); - let id4 = yield backend.registerField(mID, "id4", backend.FIELD_DAILY_LAST_NUMERIC); - let id5 = yield backend.registerField(mID, "id5", backend.FIELD_DAILY_LAST_TEXT); - let id6 = yield backend.registerField(mID, "id6", backend.FIELD_LAST_NUMERIC); - let id7 = yield backend.registerField(mID, "id7", backend.FIELD_LAST_TEXT); - - let now = new Date(); - let yesterday = new Date(now.getTime() - MILLISECONDS_PER_DAY); - - yield backend.incrementDailyCounterFromFieldID(id1, now); - yield backend.addDailyDiscreteNumericFromFieldID(id2, 3, now); - yield backend.addDailyDiscreteNumericFromFieldID(id2, 4, now); - yield backend.addDailyDiscreteNumericFromFieldID(id2, 5, yesterday); - yield backend.addDailyDiscreteNumericFromFieldID(id2, 6, yesterday); - yield backend.addDailyDiscreteTextFromFieldID(id3, "1", now); - yield backend.addDailyDiscreteTextFromFieldID(id3, "2", now); - yield backend.addDailyDiscreteTextFromFieldID(id3, "3", yesterday); - yield backend.addDailyDiscreteTextFromFieldID(id3, "4", yesterday); - yield backend.setDailyLastNumericFromFieldID(id4, 1, now); - yield backend.setDailyLastNumericFromFieldID(id4, 2, yesterday); - yield backend.setDailyLastTextFromFieldID(id5, "foo", now); - yield backend.setDailyLastTextFromFieldID(id5, "bar", yesterday); - yield backend.setLastNumericFromFieldID(id6, 42, now); - yield backend.setLastTextFromFieldID(id7, "foo", now); - - let fields = yield backend.getMeasurementValues(mID); - do_check_eq(Object.keys(fields).length, 2); - do_check_true("days" in fields); - do_check_true("singular" in fields); - do_check_eq(fields.days.size, 2); - do_check_true(fields.days.hasDay(now)); - do_check_true(fields.days.hasDay(yesterday)); - do_check_eq(fields.days.getDay(now).size, 5); - do_check_eq(fields.days.getDay(yesterday).size, 4); - do_check_eq(fields.days.getDay(now).get("id3")[0], 1); - do_check_eq(fields.days.getDay(yesterday).get("id4"), 2); - do_check_eq(fields.singular.size, 2); - do_check_eq(fields.singular.get("id6")[1], 42); - do_check_eq(fields.singular.get("id7")[1], "foo"); - - yield backend.close(); -}); - diff --git a/services/metrics/tests/xpcshell/xpcshell.ini b/services/metrics/tests/xpcshell/xpcshell.ini deleted file mode 100644 index 9707b3de7b..0000000000 --- a/services/metrics/tests/xpcshell/xpcshell.ini +++ /dev/null @@ -1,9 +0,0 @@ -[DEFAULT] -head = head.js -tail = -skip-if = toolkit == 'android' || toolkit == 'gonk' - -[test_load_modules.js] -[test_metrics_provider.js] -[test_metrics_provider_manager.js] -[test_metrics_storage.js] diff --git a/services/moz.build b/services/moz.build index b259cae80c..0193daab71 100644 --- a/services/moz.build +++ b/services/moz.build @@ -10,9 +10,6 @@ DIRS += [ 'fxaccounts', ] -if CONFIG['MOZ_SERVICES_METRICS']: - DIRS += ['metrics'] - if CONFIG['MOZ_SERVICES_SYNC']: DIRS += ['sync'] @@ -21,5 +18,3 @@ if CONFIG['MOZ_B2G'] or CONFIG['MOZ_B2GDROID']: if CONFIG['MOZ_SERVICES_CLOUDSYNC']: DIRS += ['cloudsync'] - -SPHINX_TREES['services'] = 'docs' diff --git a/toolkit/components/jsdownloads/src/DownloadCore.jsm b/toolkit/components/jsdownloads/src/DownloadCore.jsm index 9938b64b80..558d1300f0 100644 --- a/toolkit/components/jsdownloads/src/DownloadCore.jsm +++ b/toolkit/components/jsdownloads/src/DownloadCore.jsm @@ -460,6 +460,12 @@ this.Download.prototype = { throw new DownloadError({ becauseBlockedByParentalControls: true }); } + // Disallow download if needed runtime permissions have not been granted + // by user. + if (yield DownloadIntegration.shouldBlockForRuntimePermissions()) { + throw new DownloadError({ becauseBlockedByRuntimePermissions: true }); + } + // We should check if we have been canceled in the meantime, after all // the previous asynchronous operations have been executed and just // before we call the "execute" method of the saver. @@ -1495,7 +1501,8 @@ this.DownloadError = function (aProperties) this.message = aProperties.message; } else if (aProperties.becauseBlocked || aProperties.becauseBlockedByParentalControls || - aProperties.becauseBlockedByReputationCheck) { + aProperties.becauseBlockedByReputationCheck || + aProperties.becauseBlockedByRuntimePermissions) { this.message = "Download blocked."; } else { let exception = new Components.Exception("", this.result); @@ -1522,6 +1529,9 @@ this.DownloadError = function (aProperties) } else if (aProperties.becauseBlockedByReputationCheck) { this.becauseBlocked = true; this.becauseBlockedByReputationCheck = true; + } else if (aProperties.becauseBlockedByRuntimePermissions) { + this.becauseBlocked = true; + this.becauseBlockedByRuntimePermissions = true; } else if (aProperties.becauseBlocked) { this.becauseBlocked = true; } @@ -1569,6 +1579,15 @@ this.DownloadError.prototype = { */ becauseBlockedByReputationCheck: false, + /** + * Indicates the download was blocked because a runtime permission required to + * download files was not granted. + * + * This does not apply to all systems. On Android this flag is set to true if + * a needed runtime permission (storage) has not been granted by the user. + */ + becauseBlockedByRuntimePermissions: false, + /** * If this DownloadError was caused by an exception this property will * contain the original exception. This will not be serialized when saving @@ -1591,6 +1610,7 @@ this.DownloadError.prototype = { becauseBlocked: this.becauseBlocked, becauseBlockedByParentalControls: this.becauseBlockedByParentalControls, becauseBlockedByReputationCheck: this.becauseBlockedByReputationCheck, + becauseBlockedByRuntimePermissions: this.becauseBlockedByRuntimePermissions, }; serializeUnknownProperties(this, serializable); @@ -1615,7 +1635,8 @@ this.DownloadError.fromSerializable = function (aSerializable) { property != "becauseTargetFailed" && property != "becauseBlocked" && property != "becauseBlockedByParentalControls" && - property != "becauseBlockedByReputationCheck"); + property != "becauseBlockedByReputationCheck" && + property != "becauseBlockedByRuntimePermissions"); return e; }; @@ -2080,6 +2101,9 @@ this.DownloadCopySaver.prototype = { // In case an error occurs while setting up the chain of objects for // the download, ensure that we release the resources of the saver. backgroundFileSaver.finish(Cr.NS_ERROR_FAILURE); + // Since we're not going to handle deferSaveComplete.promise below, + // we need to make sure that the rejection is handled. + deferSaveComplete.promise.catch(() => {}); throw ex; } @@ -2316,14 +2340,18 @@ this.DownloadLegacySaver.prototype = { */ onProgressBytes: function DLS_onProgressBytes(aCurrentBytes, aTotalBytes) { + this.progressWasNotified = true; + // Ignore progress notifications until we are ready to process them. if (!this.setProgressBytesFn) { + // Keep the data from the last progress notification that was received. + this.currentBytes = aCurrentBytes; + this.totalBytes = aTotalBytes; return; } let hasPartFile = !!this.download.target.partFilePath; - this.progressWasNotified = true; this.setProgressBytesFn(aCurrentBytes, aTotalBytes, aCurrentBytes > 0 && hasPartFile); }, @@ -2433,6 +2461,9 @@ this.DownloadLegacySaver.prototype = { } this.setProgressBytesFn = aSetProgressBytesFn; + if (this.progressWasNotified) { + this.onProgressBytes(this.currentBytes, this.totalBytes); + } return Task.spawn(function* task_DLS_execute() { try { diff --git a/toolkit/components/jsdownloads/src/DownloadImport.jsm b/toolkit/components/jsdownloads/src/DownloadImport.jsm index ba85761c0e..6c175c0239 100644 --- a/toolkit/components/jsdownloads/src/DownloadImport.jsm +++ b/toolkit/components/jsdownloads/src/DownloadImport.jsm @@ -174,7 +174,7 @@ this.DownloadImport.prototype = { yield this.list.add(download); if (resumeDownload) { - download.start(); + download.start().catch(() => {}); } else { yield download.refresh(); } diff --git a/toolkit/components/jsdownloads/src/DownloadIntegration.jsm b/toolkit/components/jsdownloads/src/DownloadIntegration.jsm index 58b3bcbcf1..404f07819d 100644 --- a/toolkit/components/jsdownloads/src/DownloadIntegration.jsm +++ b/toolkit/components/jsdownloads/src/DownloadIntegration.jsm @@ -68,6 +68,10 @@ XPCOMUtils.defineLazyServiceGetter(this, "gMIMEService", XPCOMUtils.defineLazyServiceGetter(this, "gExternalProtocolService", "@mozilla.org/uriloader/external-protocol-service;1", "nsIExternalProtocolService"); +#ifdef MOZ_WIDGET_ANDROID +XPCOMUtils.defineLazyModuleGetter(this, "RuntimePermissions", + "resource://gre/modules/RuntimePermissions.jsm"); +#endif XPCOMUtils.defineLazyGetter(this, "gParentalControlsService", function() { if ("@mozilla.org/parental-controls-service;1" in Cc) { @@ -136,7 +140,13 @@ this.DownloadIntegration = { dontLoadObservers: false, dontCheckParentalControls: false, shouldBlockInTest: false, + dontCheckRuntimePermissions: false, + shouldBlockInTestForRuntimePermissions: false, +#ifdef MOZ_URL_CLASSIFIER + dontCheckApplicationReputation: false, +#else dontCheckApplicationReputation: true, +#endif shouldBlockInTestForApplicationReputation: false, shouldKeepBlockedDataInTest: false, dontOpenFileAndFolder: false, @@ -492,6 +502,25 @@ this.DownloadIntegration = { return Promise.resolve(shouldBlock); }, + /** + * Checks to determine whether to block downloads for not granted runtime permissions. + * + * @return {Promise} + * @resolves The boolean indicates to block downloads or not. + */ + shouldBlockForRuntimePermissions: function DI_shouldBlockForRuntimePermissions() { + if (this.dontCheckRuntimePermissions) { + return Promise.resolve(this.shouldBlockInTestForRuntimePermissions); + } + +#ifdef MOZ_WIDGET_ANDROID + return RuntimePermissions.waitForPermissions(RuntimePermissions.WRITE_EXTERNAL_STORAGE) + .then(permissionGranted => !permissionGranted); +#else + return Promise.resolve(false); +#endif + }, + /** * Checks to determine whether to block downloads because they might be * malware, based on application reputation checks. @@ -928,6 +957,20 @@ this.DownloadIntegration = { return Promise.resolve(); }, + /** + * Force a save on _store if it exists. Used to ensure downloads do not + * persist after being sanitized on Android. + * + * @return {Promise} + * @resolves When _store.save() completes. + */ + forceSave: function DI_forceSave() { + if (this._store) { + return this._store.save(); + } + return Promise.resolve(); + }, + /** * Checks if we have already imported (or attempted to import) * the downloads database from the previous SQLite storage. @@ -1054,7 +1097,7 @@ this.DownloadObserver = { this._wakeTimer = null; for (let download of this._canceledOfflineDownloads) { - download.start(); + download.start().catch(() => {}); } }, diff --git a/toolkit/components/jsdownloads/src/DownloadLegacy.js b/toolkit/components/jsdownloads/src/DownloadLegacy.js index 48218d87ad..c4b6380855 100644 --- a/toolkit/components/jsdownloads/src/DownloadLegacy.js +++ b/toolkit/components/jsdownloads/src/DownloadLegacy.js @@ -243,7 +243,7 @@ DownloadLegacyTransfer.prototype = { } // Start the download before allowing it to be controlled. Ignore errors. - aDownload.start().then(null, () => {}); + aDownload.start().catch(() => {}); // Start processing all the other events received through nsITransfer. this._deferDownload.resolve(aDownload); diff --git a/toolkit/components/jsdownloads/src/DownloadStore.jsm b/toolkit/components/jsdownloads/src/DownloadStore.jsm index 8ad7e720d6..83bc2824fb 100644 --- a/toolkit/components/jsdownloads/src/DownloadStore.jsm +++ b/toolkit/components/jsdownloads/src/DownloadStore.jsm @@ -124,8 +124,8 @@ this.DownloadStore.prototype = { try { if (!download.succeeded && !download.canceled && !download.error) { // Try to restart the download if it was in progress during the - // previous session. - download.start(); + // previous session. Ignore errors. + download.start().catch(() => {}); } else { // If the download was not in progress, try to update the current // progress from disk. This is relevant in case we retained diff --git a/toolkit/components/jsdownloads/test/browser/browser_DownloadPDFSaver.js b/toolkit/components/jsdownloads/test/browser/browser_DownloadPDFSaver.js index 63e1174656..e08e500f78 100644 --- a/toolkit/components/jsdownloads/test/browser/browser_DownloadPDFSaver.js +++ b/toolkit/components/jsdownloads/test/browser/browser_DownloadPDFSaver.js @@ -88,7 +88,7 @@ add_task(function* test_cancel_pdf_download() { }); yield test_download_windowRef(tab, download); - download.start(); + download.start().catch(() => {}); // Immediately cancel the download to test that it is erased correctly. yield download.cancel(); diff --git a/toolkit/components/jsdownloads/test/unit/common_test_Download.js b/toolkit/components/jsdownloads/test/unit/common_test_Download.js index 752743af9a..d382f645c1 100644 --- a/toolkit/components/jsdownloads/test/unit/common_test_Download.js +++ b/toolkit/components/jsdownloads/test/unit/common_test_Download.js @@ -32,7 +32,7 @@ function promiseStartDownload(aSourceUrl) { } return promiseNewDownload(aSourceUrl).then(download => { - download.start(); + download.start().catch(() => {}); return download; }); } @@ -64,7 +64,7 @@ function promiseStartDownload_tryToKeepPartialData() { partFilePath: targetFilePath + ".part" }, }); download.tryToKeepPartialData = true; - download.start(); + download.start().catch(() => {}); } else { // Start a download using nsIExternalHelperAppService, that is configured // to keep partially downloaded data by default. @@ -435,7 +435,7 @@ add_task(function* test_empty_progress_tryToKeepPartialData() partFilePath: targetFilePath + ".part" }, }); download.tryToKeepPartialData = true; - download.start(); + download.start().catch(() => {}); } else { // Start a download using nsIExternalHelperAppService, that is configured // to keep partially downloaded data by default. @@ -491,7 +491,7 @@ add_task(function* test_empty_noprogress() } }; - download.start(); + download.start().catch(() => {}); } else { // When testing DownloadLegacySaver, the download is already started when it // is created, and it may have already made all needed property change @@ -856,7 +856,7 @@ add_task(function* test_cancel_midway_restart_tryToKeepPartialData_false() // Restart the download from the beginning. mustInterruptResponses(); - download.start(); + download.start().catch(() => {}); yield promiseDownloadMidway(download); yield promisePartFileReady(download); @@ -1143,7 +1143,7 @@ add_task(function* test_whenSucceeded_after_restart() // we can verify getting a reference before the first download attempt. download = yield promiseNewDownload(httpUrl("interruptible.txt")); promiseSucceeded = download.whenSucceeded(); - download.start(); + download.start().catch(() => {}); } else { // When testing DownloadLegacySaver, the download is already started when it // is created, thus we cannot get the reference before the first attempt. @@ -1156,7 +1156,7 @@ add_task(function* test_whenSucceeded_after_restart() // The second request is allowed to complete. continueResponses(); - download.start(); + download.start().catch(() => {}); // Wait for the download to finish by waiting on the whenSucceeded promise. yield promiseSucceeded; @@ -1343,7 +1343,7 @@ add_task(function* test_error_restart() source: httpUrl("source.txt"), target: targetFile, }); - download.start(); + download.start().catch(() => {}); } else { download = yield promiseStartLegacyDownload(null, { targetFile: targetFile }); @@ -1640,6 +1640,47 @@ add_task(function* test_blocked_parental_controls_httpstatus450() do_check_false(yield OS.File.exists(download.target.path)); }); +/** + * Download with runtime permissions + */ +add_task(function* test_blocked_runtime_permissions() +{ + function cleanup() { + DownloadIntegration.shouldBlockInTestForRuntimePermissions = false; + } + do_register_cleanup(cleanup); + DownloadIntegration.shouldBlockInTestForRuntimePermissions = true; + + let download; + try { + if (!gUseLegacySaver) { + // When testing DownloadCopySaver, we want to check that the promise + // returned by the "start" method is rejected. + download = yield promiseNewDownload(); + yield download.start(); + } else { + // When testing DownloadLegacySaver, we cannot be sure whether we are + // testing the promise returned by the "start" method or we are testing + // the "error" property checked by promiseDownloadStopped. This happens + // because we don't have control over when the download is started. + download = yield promiseStartLegacyDownload(); + yield promiseDownloadStopped(download); + } + do_throw("The download should have blocked."); + } catch (ex) { + if (!(ex instanceof Downloads.Error) || !ex.becauseBlocked) { + throw ex; + } + do_check_true(ex.becauseBlockedByRuntimePermissions); + do_check_true(download.error.becauseBlockedByRuntimePermissions); + } + + // Now that the download stopped, the target file should not exist. + do_check_false(yield OS.File.exists(download.target.path)); + + cleanup(); +}); + /** * Check that DownloadCopySaver can always retrieve the hash. * DownloadLegacySaver can only retrieve the hash when @@ -2145,7 +2186,7 @@ add_task(function* test_platform_integration() source: httpUrl("source.txt"), target: targetFile, }); - download.start(); + download.start().catch(() => {}); } // Wait for the whenSucceeded promise to be resolved first. diff --git a/toolkit/components/jsdownloads/test/unit/head.js b/toolkit/components/jsdownloads/test/unit/head.js index 3546c6124c..fe505fcc6d 100644 --- a/toolkit/components/jsdownloads/test/unit/head.js +++ b/toolkit/components/jsdownloads/test/unit/head.js @@ -788,6 +788,8 @@ add_task(function test_common_initialize() DownloadIntegration.dontOpenFileAndFolder = true; DownloadIntegration._deferTestOpenFile = Promise.defer(); DownloadIntegration._deferTestShowDir = Promise.defer(); + // Disable checking runtime permissions. + DownloadIntegration.dontCheckRuntimePermissions = true; // Avoid leaking uncaught promise errors DownloadIntegration._deferTestOpenFile.promise.then(null, () => undefined); diff --git a/toolkit/components/jsdownloads/test/unit/test_DownloadIntegration.js b/toolkit/components/jsdownloads/test/unit/test_DownloadIntegration.js index d5500ff366..82f3cffa04 100644 --- a/toolkit/components/jsdownloads/test/unit/test_DownloadIntegration.js +++ b/toolkit/components/jsdownloads/test/unit/test_DownloadIntegration.js @@ -215,7 +215,7 @@ add_task(function* test_notifications() let download3 = yield promiseNewDownload(httpUrl("interruptible.txt")); let promiseAttempt1 = download1.start(); let promiseAttempt2 = download2.start(); - download3.start(); + download3.start().catch(() => {}); // Add downloads to list. yield list.add(download1); @@ -250,8 +250,8 @@ add_task(function* test_no_notifications() let list = yield promiseNewList(isPrivate); let download1 = yield promiseNewDownload(httpUrl("interruptible.txt")); let download2 = yield promiseNewDownload(httpUrl("interruptible.txt")); - download1.start(); - download2.start(); + download1.start().catch(() => {}); + download2.start().catch(() => {}); // Add downloads to list. yield list.add(download1); @@ -316,7 +316,7 @@ add_task(function* test_suspend_resume() { return Task.spawn(function* () { let download = yield promiseNewDownload(httpUrl("interruptible.txt")); - download.start(); + download.start().catch(() => {}); list.add(download); return download; }); diff --git a/toolkit/components/jsdownloads/test/unit/test_DownloadList.js b/toolkit/components/jsdownloads/test/unit/test_DownloadList.js index 8eda8592f8..9cbeabc004 100644 --- a/toolkit/components/jsdownloads/test/unit/test_DownloadList.js +++ b/toolkit/components/jsdownloads/test/unit/test_DownloadList.js @@ -348,7 +348,7 @@ add_task(function* test_history_expiration() // Work with one finished download and one canceled download. yield downloadOne.start(); - downloadTwo.start(); + downloadTwo.start().catch(() => {}); yield downloadTwo.cancel(); // We must replace the visits added while executing the downloads with visits @@ -471,7 +471,7 @@ add_task(function* test_DownloadSummary() // Add a public download that has been canceled midway. let canceledPublicDownload = yield promiseNewDownload(httpUrl("interruptible.txt")); - canceledPublicDownload.start(); + canceledPublicDownload.start().catch(() => {}); yield promiseDownloadMidway(canceledPublicDownload); yield canceledPublicDownload.cancel(); yield publicList.add(canceledPublicDownload); @@ -479,7 +479,7 @@ add_task(function* test_DownloadSummary() // Add a public download that is in progress. let inProgressPublicDownload = yield promiseNewDownload(httpUrl("interruptible.txt")); - inProgressPublicDownload.start(); + inProgressPublicDownload.start().catch(() => {}); yield promiseDownloadMidway(inProgressPublicDownload); yield publicList.add(inProgressPublicDownload); @@ -488,7 +488,7 @@ add_task(function* test_DownloadSummary() source: { url: httpUrl("interruptible.txt"), isPrivate: true }, target: getTempFile(TEST_TARGET_FILE_NAME).path, }); - inProgressPrivateDownload.start(); + inProgressPrivateDownload.start().catch(() => {}); yield promiseDownloadMidway(inProgressPrivateDownload); yield privateList.add(inProgressPrivateDownload); diff --git a/toolkit/components/search/nsSearchService.js b/toolkit/components/search/nsSearchService.js index 669c405467..e93ab5c9b0 100644 --- a/toolkit/components/search/nsSearchService.js +++ b/toolkit/components/search/nsSearchService.js @@ -1006,6 +1006,10 @@ EngineURL.prototype = { // (purpose="") work consistently rather than having to define "null" and "" purposes. var purpose = aPurpose || ""; + // If the 'system' purpose isn't defined in the plugin, fallback to 'searchbar'. + if (purpose == "system" && !this.params.some(p => p.purpose == "system")) + purpose = "searchbar"; + // Create an application/x-www-form-urlencoded representation of our params // (name=value&name=value&name=value) var dataString = ""; @@ -4202,6 +4206,13 @@ SearchService.prototype = { }, _addObservers: function SRCH_SVC_addObservers() { + if (this._observersAdded) { + // There might be a race between synchronous and asynchronous + // initialization for which we try to register the observers twice. + return; + } + this._observersAdded = true; + Services.obs.addObserver(this, SEARCH_ENGINE_TOPIC, false); Services.obs.addObserver(this, QUIT_APPLICATION_TOPIC, false); @@ -4249,6 +4260,7 @@ SearchService.prototype = { () => shutdownState ); }, + _observersAdded: false, _removeObservers: function SRCH_SVC_removeObservers() { Services.obs.removeObserver(this, SEARCH_ENGINE_TOPIC); diff --git a/toolkit/components/search/tests/xpcshell/data/engine-system-purpose.xml b/toolkit/components/search/tests/xpcshell/data/engine-system-purpose.xml new file mode 100644 index 0000000000..57ecd32d78 --- /dev/null +++ b/toolkit/components/search/tests/xpcshell/data/engine-system-purpose.xml @@ -0,0 +1,10 @@ + + +engine-system-purpose + + + + + + + diff --git a/toolkit/components/search/tests/xpcshell/test_purpose.js b/toolkit/components/search/tests/xpcshell/test_purpose.js index 56c26de8a0..f38e76648f 100644 --- a/toolkit/components/search/tests/xpcshell/test_purpose.js +++ b/toolkit/components/search/tests/xpcshell/test_purpose.js @@ -54,5 +54,16 @@ add_task(function* test_purpose() { check_submission("&channel=sb", "", null, "searchbar"); check_submission("&channel=sb", "", "text/html", "searchbar"); + // verify that the 'system' purpose falls back to the 'searchbar' purpose. + base = "http://www.google.com/search?q=foo"; + check_submission("&channel=sb", "foo", "text/html", "system"); + check_submission("&channel=sb", "foo", "text/html", "searchbar"); + // Add an engine that actually defines the 'system' purpose... + [engine] = yield addTestEngines([ + { name: "engine-system-purpose", xmlFileName: "engine-system-purpose.xml" } + ]); + // ... and check that the system purpose is used correctly. + check_submission("&channel=sys", "foo", "text/html", "system"); + do_test_finished(); }); diff --git a/toolkit/components/search/tests/xpcshell/xpcshell.ini b/toolkit/components/search/tests/xpcshell/xpcshell.ini index 05e345f176..2a00fd955a 100644 --- a/toolkit/components/search/tests/xpcshell/xpcshell.ini +++ b/toolkit/components/search/tests/xpcshell/xpcshell.ini @@ -13,6 +13,7 @@ support-files = data/engine-rel-searchform.xml data/engine-rel-searchform-post.xml data/engine-rel-searchform-purpose.xml + data/engine-system-purpose.xml data/engineImages.xml data/ico-size-16x16-png.ico data/invalid-engine.xml diff --git a/toolkit/modules/AppConstants.jsm b/toolkit/modules/AppConstants.jsm index 514f530507..e295b55bca 100644 --- a/toolkit/modules/AppConstants.jsm +++ b/toolkit/modules/AppConstants.jsm @@ -5,6 +5,9 @@ "use strict"; +Components.utils.import("resource://gre/modules/XPCOMUtils.jsm"); +XPCOMUtils.defineLazyModuleGetter(this, "Services", "resource://gre/modules/Services.jsm"); + this.EXPORTED_SYMBOLS = ["AppConstants"]; // Immutable for export. @@ -122,6 +125,18 @@ let AppConstants = Object.freeze({ "other", #endif + isPlatformAndVersionAtLeast(platform, version) { + let platformVersion = Services.sysinfo.getProperty("version"); + return platform == this.platform && + Services.vc.compare(platformVersion, version) >= 0; + }, + + isPlatformAndVersionAtMost(platform, version) { + let platformVersion = Services.sysinfo.getProperty("version"); + return platform == this.platform && + Services.vc.compare(platformVersion, version) <= 0; + }, + MOZ_CRASHREPORTER: #ifdef MOZ_CRASHREPORTER true, diff --git a/toolkit/modules/moz.build b/toolkit/modules/moz.build index bc4dc18dfa..fae9469432 100644 --- a/toolkit/modules/moz.build +++ b/toolkit/modules/moz.build @@ -9,6 +9,10 @@ BROWSER_CHROME_MANIFESTS += ['tests/browser/browser.ini'] MOCHITEST_MANIFESTS += ['tests/mochitest/mochitest.ini'] MOCHITEST_CHROME_MANIFESTS += ['tests/chrome/chrome.ini'] +TESTING_JS_MODULES += [ + 'tests/PromiseTestUtils.jsm', +] + SPHINX_TREES['toolkit_modules'] = 'docs' EXTRA_JS_MODULES += [ diff --git a/toolkit/modules/tests/PromiseTestUtils.jsm b/toolkit/modules/tests/PromiseTestUtils.jsm new file mode 100644 index 0000000000..d60b785a58 --- /dev/null +++ b/toolkit/modules/tests/PromiseTestUtils.jsm @@ -0,0 +1,241 @@ +/* Any copyright is dedicated to the Public Domain. + * http://creativecommons.org/publicdomain/zero/1.0/ */ + +/* + * Detects and reports unhandled rejections during test runs. Test harnesses + * will fail tests in this case, unless the test whitelists itself. + */ + +"use strict"; + +this.EXPORTED_SYMBOLS = [ + "PromiseTestUtils", +]; + +const { classes: Cc, interfaces: Ci, utils: Cu, results: Cr } = Components; + +Cu.import("resource://gre/modules/Services.jsm", this); + +// Keep "JSMPromise" separate so "Promise" still refers to DOM Promises. +let JSMPromise = Cu.import("resource://gre/modules/Promise.jsm", {}).Promise; + +// For now, we need test harnesses to provide a reference to Assert.jsm. +let Assert = null; + +this.PromiseTestUtils = { + /** + * Array of objects containing the details of the Promise rejections that are + * currently left uncaught. This includes DOM Promise and Promise.jsm. When + * rejections in DOM Promises are consumed, they are removed from this list. + * + * The objects contain at least the following properties: + * { + * message: The error message associated with the rejection, if any. + * date: Date object indicating when the rejection was observed. + * id: For DOM Promise only, the Promise ID from PromiseDebugging. This is + * only used for tracking and should not be checked by the callers. + * stack: nsIStackFrame, SavedFrame, or string indicating the stack at the + * time the rejection was triggered. May also be null if the + * rejection was triggered while a script was on the stack. + * } + */ + _rejections: [], + + /** + * When an uncaught rejection is detected, it is ignored if one of the + * functions in this array returns true when called with the rejection details + * as its only argument. When a function matches an expected rejection, it is + * then removed from the array. + */ + _rejectionIgnoreFns: [], + + /** + * Called only by the test infrastructure, registers the rejection observers. + * + * This should be called only once, and a matching "uninit" call must be made + * or the tests will crash on shutdown. + */ + init() { + if (this._initialized) { + Cu.reportError("This object was already initialized."); + return; + } + + PromiseDebugging.addUncaughtRejectionObserver(this); + + // Promise.jsm rejections are only reported to this observer when requested, + // so we don't have to store a key to remove them when consumed. + JSMPromise.Debugging.addUncaughtErrorObserver( + rejection => this._rejections.push(rejection)); + + this._initialized = true; + }, + _initialized: false, + + /** + * Called only by the test infrastructure, unregisters the observers. + */ + uninit() { + if (!this._initialized) { + return; + } + + PromiseDebugging.removeUncaughtRejectionObserver(this); + JSMPromise.Debugging.clearUncaughtErrorObservers(); + + this._initialized = false; + }, + + /** + * Called only by the test infrastructure, spins the event loop until the + * messages for pending DOM Promise rejections have been processed. + */ + ensureDOMPromiseRejectionsProcessed() { + let observed = false; + let observer = { + onLeftUncaught: promise => { + if (PromiseDebugging.getState(promise).reason === + this._ensureDOMPromiseRejectionsProcessedReason) { + observed = true; + } + }, + onConsumed() {}, + }; + + PromiseDebugging.addUncaughtRejectionObserver(observer); + Promise.reject(this._ensureDOMPromiseRejectionsProcessedReason); + while (!observed) { + Services.tm.mainThread.processNextEvent(true); + } + PromiseDebugging.removeUncaughtRejectionObserver(observer); + }, + _ensureDOMPromiseRejectionsProcessedReason: {}, + + /** + * Called only by the tests for PromiseDebugging.addUncaughtRejectionObserver + * and for JSMPromise.Debugging, disables the observers in this module. + */ + disableUncaughtRejectionObserverForSelfTest() { + this.uninit(); + }, + + /** + * Called by tests that have been whitelisted, disables the observers in this + * module. For new tests where uncaught rejections are expected, you should + * use the more granular expectUncaughtRejection function instead. + */ + thisTestLeaksUncaughtRejectionsAndShouldBeFixed() { + this.uninit(); + }, + + /** + * Sets or updates the Assert object instance to be used for error reporting. + */ + set Assert(assert) { + Assert = assert; + }, + + // UncaughtRejectionObserver + onLeftUncaught(promise) { + let message = "(Unable to convert rejection reason to string.)"; + try { + let reason = PromiseDebugging.getState(promise).reason; + if (reason === this._ensureDOMPromiseRejectionsProcessedReason) { + // Ignore the special promise for ensureDOMPromiseRejectionsProcessed. + return; + } + message = reason.message || ("" + reason); + } catch (ex) {} + + // It's important that we don't store any reference to the provided Promise + // object or its value after this function returns in order to avoid leaks. + this._rejections.push({ + id: PromiseDebugging.getPromiseID(promise), + message, + date: new Date(), + stack: PromiseDebugging.getRejectionStack(promise), + }); + }, + + // UncaughtRejectionObserver + onConsumed(promise) { + // We don't expect that many unhandled rejections will appear at the same + // time, so the algorithm doesn't need to be optimized for that case. + let id = PromiseDebugging.getPromiseID(promise); + let index = this._rejections.findIndex(rejection => rejection.id == id); + // If we get a consumption notification for a rejection that was left + // uncaught before this module was initialized, we can safely ignore it. + if (index != -1) { + this._rejections.splice(index, 1); + } + }, + + /** + * Informs the test suite that the test code will generate a Promise rejection + * that will still be unhandled when the test file terminates. + * + * This method must be called once for each instance of Promise that is + * expected to be uncaught, even if the rejection reason is the same for each + * instance. + * + * If the expected rejection does not occur, the test will fail. + * + * @param regExpOrCheckFn + * This can either be a regular expression that should match the error + * message of the rejection, or a check function that is invoked with + * the rejection details object as its first argument. + */ + expectUncaughtRejection(regExpOrCheckFn) { + let checkFn = !("test" in regExpOrCheckFn) ? regExpOrCheckFn : + rejection => regExpOrCheckFn.test(rejection.message); + this._rejectionIgnoreFns.push(checkFn); + }, + + /** + * Fails the test if there are any uncaught rejections at this time that have + * not been whitelisted using expectUncaughtRejection. + * + * Depending on the configuration of the test suite, this function might only + * report the details of the first uncaught rejection that was generated. + * + * This is called by the test suite at the end of each test function. + */ + assertNoUncaughtRejections() { + // Ask Promise.jsm to report all uncaught rejections to the observer now. + JSMPromise.Debugging.flushUncaughtErrors(); + + // If there is any uncaught rejection left at this point, the test fails. + while (this._rejections.length > 0) { + let rejection = this._rejections.shift(); + + // If one of the ignore functions matches, ignore the rejection, then + // remove the function so that each function only matches one rejection. + let index = this._rejectionIgnoreFns.findIndex(f => f(rejection)); + if (index != -1) { + this._rejectionIgnoreFns.splice(index, 1); + continue; + } + + // Report the error. This operation can throw an exception, depending on + // the configuration of the test suite that handles the assertion. + Assert.ok(false, + `A promise chain failed to handle a rejection:` + + ` ${rejection.message} - rejection date: ${rejection.date}`+ + ` - stack: ${rejection.stack}`); + } + }, + + /** + * Fails the test if any rejection indicated by expectUncaughtRejection has + * not yet been reported at this time. + * + * This is called by the test suite at the end of each test file. + */ + assertNoMoreExpectedRejections() { + // Only log this condition is there is a failure. + if (this._rejectionIgnoreFns.length > 0) { + Assert.equal(this._rejectionIgnoreFns.length, 0, + "Unable to find a rejection expected by expectUncaughtRejection."); + } + }, +}; diff --git a/toolkit/modules/tests/xpcshell/test_Promise.js b/toolkit/modules/tests/xpcshell/test_Promise.js index de1ea022ae..7a4bf8d59f 100644 --- a/toolkit/modules/tests/xpcshell/test_Promise.js +++ b/toolkit/modules/tests/xpcshell/test_Promise.js @@ -5,10 +5,10 @@ Components.utils.import("resource://gre/modules/Promise.jsm"); Components.utils.import("resource://gre/modules/Services.jsm"); Components.utils.import("resource://gre/modules/Task.jsm"); +Components.utils.import("resource://testing-common/PromiseTestUtils.jsm"); -// Deactivate the standard xpcshell observer, as it turns uncaught -// rejections into failures, which we don't want here. -Promise.Debugging.clearUncaughtErrorObservers(); +// Prevent test failures due to the unhandled rejections in this test file. +PromiseTestUtils.disableUncaughtRejectionObserverForSelfTest(); //////////////////////////////////////////////////////////////////////////////// //// Test runner diff --git a/toolkit/modules/tests/xpcshell/test_PromiseUtils.js b/toolkit/modules/tests/xpcshell/test_PromiseUtils.js index ff9ba3688b..5f1ac8b7d0 100644 --- a/toolkit/modules/tests/xpcshell/test_PromiseUtils.js +++ b/toolkit/modules/tests/xpcshell/test_PromiseUtils.js @@ -5,6 +5,7 @@ Components.utils.import("resource://gre/modules/PromiseUtils.jsm"); Components.utils.import("resource://gre/modules/Timer.jsm"); +Components.utils.import("resource://testing-common/PromiseTestUtils.jsm"); // Tests for PromiseUtils.jsm function run_test() { @@ -98,8 +99,9 @@ add_task(function* test_reject_resolved_promise() { /* Test for the case when a rejected Promise is * passed to the reject method */ add_task(function* test_reject_resolved_promise() { + PromiseTestUtils.expectUncaughtRejection(/This one rejects/); let def = PromiseUtils.defer(); - let p = new Promise((resolve, reject) => reject(new Error("This on rejects"))); + let p = new Promise((resolve, reject) => reject(new Error("This one rejects"))); def.reject(p); yield Assert.rejects(def.promise, Promise, "Rejection with a rejected promise uses the passed promise itself as the reason of rejection"); }); diff --git a/view/nsView.cpp b/view/nsView.cpp index 033f88fbc5..61208d02bb 100644 --- a/view/nsView.cpp +++ b/view/nsView.cpp @@ -1086,7 +1086,9 @@ nsView::DidCompositeWindow(const TimeStamp& aCompositeStart, nsAutoScriptBlocker scriptBlocker; nsPresContext* context = presShell->GetPresContext(); - context->GetRootPresContext()->NotifyDidPaintForSubtree(nsIPresShell::PAINT_COMPOSITE); + nsRootPresContext* rootContext = context->GetRootPresContext(); + MOZ_ASSERT(rootContext, "rootContext must be valid."); + rootContext->NotifyDidPaintForSubtree(nsIPresShell::PAINT_COMPOSITE); // If the two timestamps are identical, this was likely a fake composite // event which wouldn't be terribly useful to display. diff --git a/xpcom/tests/unit/xpcshell.ini b/xpcom/tests/unit/xpcshell.ini index 858488a5ef..371b8a846a 100644 --- a/xpcom/tests/unit/xpcshell.ini +++ b/xpcom/tests/unit/xpcshell.ini @@ -33,6 +33,8 @@ fail-if = os == "android" fail-if = os == "android" [test_file_createUnique.js] [test_file_equality.js] +# Bug 1144393: fails consistently on Android 4.3 emulator +fail-if = android_version == "18" [test_hidden_files.js] [test_home.js] # Bug 676998: test fails consistently on Android diff --git a/xulrunner/confvars.sh b/xulrunner/confvars.sh index a115bda166..067794098f 100755 --- a/xulrunner/confvars.sh +++ b/xulrunner/confvars.sh @@ -11,9 +11,9 @@ MOZ_CHROME_FILE_FORMAT=omni MOZ_APP_VERSION=$MOZILLA_VERSION MOZ_PLACES=1 MOZ_EXTENSIONS_DEFAULT=" gio" +MOZ_URL_CLASSIFIER=1 MOZ_SERVICES_COMMON=1 MOZ_SERVICES_CRYPTO=1 -MOZ_SERVICES_METRICS=1 MOZ_SERVICES_SYNC=1 MOZ_MEDIA_NAVIGATOR=1 MOZ_SERVICES_HEALTHREPORT=1