feat(firestore-bigquery-export): add log level config param (#2330)

* feat(firestore-bigquery-export): add log level config param

* feat(firestore-bigquery-export): added no logging option (silent)

* perf(firestore-bigquery-export): improved performance of log levels code

* refactor(firestore-bigquery-export): code shortening

* chore(firestore-bigquery-export): format and docs

* fix(firestore-bigquery-export): tests

* chore(firestore-bigquery-export): make some info logs debug logs

* chore(firestore-bigquery-export): made some changes based on feedback

* feat(firestore-bigquery-export): add tests for logger

* chore(firestore-bigquery-export): format

* chore(firestore-bigquery-export): adjust for latest changes

* chore(firestore-bigquery-export): test mocking adjustments

* chore(firestore-bigquery-export): bump extension version and add changelog

* fix(firestore-bigquery-export): converted a missed log to use new logger

---------

Co-authored-by: Jacob Cable <[email protected]>

Author: CorieW

firestore-bigquery-export/CHANGELOG.md

@@ -1,3 +1,7 @@
+## Version 0.1.60
+
+feat - configure a log level to control the verbosity of logs.
+
 ## Version 0.1.59
 
 docs - remove references to lifecycle backfill feature.

firestore-bigquery-export/README.md

@@ -297,6 +297,8 @@ Available schema extensions table fields for clustering include: `document_id, d
 
 * Maximum number of enqueue attempts: This parameter will set the maximum number of attempts to enqueue a document to cloud tasks for export to BigQuery.
 
+* Log level: The log level for the extension. The log level controls the verbosity of the extension's logs. The available log levels are: debug, info, warn, and error. To reduce the volume of logs, use a log level of warn or error.
+
 
 
 **Cloud Functions:**

firestore-bigquery-export/extension.yaml

@@ -13,7 +13,7 @@
 # limitations under the License.
 
 name: firestore-bigquery-export
-version: 0.1.59
+version: 0.1.60
 specVersion: v1beta
 
 displayName: Stream Firestore to BigQuery
@@ -461,6 +461,27 @@ params:
     validationErrorMessage: Please select an integer between 1 and 10
     default: 3
 
+  - param: LOG_LEVEL
+    label: Log level
+    description: >-
+      The log level for the extension. The log level controls the verbosity of
+      the extension's logs. The available log levels are: debug, info, warn, and
+      error. To reduce the volume of logs, use a log level of warn or error.
+    type: select
+    options:
+      - label: Debug
+        value: debug
+      - label: Info
+        value: info
+      - label: Warn
+        value: warn
+      - label: Error
+        value: error
+      - label: Silent
+        value: silent
+    default: info
+    required: true
+
 events:
   # OLD event types for backward compatibility
   - type: firebase.extensions.firestore-counter.v1.onStart

firestore-bigquery-export/functions/__tests__/__snapshots__/config.test.ts.snap

@@ -21,6 +21,7 @@ Object {
   "instanceId": undefined,
   "kmsKeyName": "test",
   "location": "us-central1",
+  "logLevel": "info",
   "maxDispatchesPerSecond": 10,
   "maxEnqueueAttempts": 3,
   "maxStaleness": undefined,

firestore-bigquery-export/functions/__tests__/functions.test.ts

@@ -16,6 +16,21 @@ jest.mock("@firebaseextensions/firestore-bigquery-change-tracker", () => ({
     UPDATE: 1,
     CREATE: 0,
   },
+  LogLevel: {
+    DEBUG: "debug",
+    INFO: "info",
+    WARN: "warn",
+    ERROR: "error",
+    SILENT: "silent",
+  },
+  Logger: jest.fn().mockImplementation(() => ({
+    debug: jest.fn(),
+    info: jest.fn(),
+    warn: jest.fn(),
+    error: jest.fn(),
+    log: jest.fn(),
+    setLogLevel: jest.fn(),
+  })),
 }));
 
 jest.mock("firebase-admin/functions", () => ({

firestore-bigquery-export/functions/package-lock.json

@@ -13,7 +13,7 @@
   "author": "Jan Wyszynski <[email protected]>",
   "license": "Apache-2.0",
   "dependencies": {
-    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.39",
+    "@firebaseextensions/firestore-bigquery-change-tracker": "^1.1.40",
     "@google-cloud/bigquery": "^7.6.0",
     "@types/chai": "^4.1.6",
     "@types/express-serve-static-core": "4.17.30",

firestore-bigquery-export/functions/package.json

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+import { LogLevel } from "@firebaseextensions/firestore-bigquery-change-tracker";
 
 function timePartitioning(type) {
   if (
@@ -73,4 +74,5 @@ export default {
   backupBucketName:
     process.env.BACKUP_GCS_BUCKET || `${process.env.PROJECT_ID}.appspot.com`,
   backupDir: `_${process.env.INSTANCE_ID || "firestore-bigquery-export"}`,
+  logLevel: process.env.LOG_LEVEL || LogLevel.INFO,
 };

firestore-bigquery-export/functions/src/config.ts

@@ -57,13 +57,15 @@ const eventTrackerConfig = {
     config.viewType === "materialized_incremental",
   maxStaleness: config.maxStaleness,
   refreshIntervalMinutes: config.refreshIntervalMinutes,
+  logLevel: config.logLevel,
 };
 
 // Initialize the Firestore Event History Tracker with the given configuration.
 const eventTracker: FirestoreBigQueryEventHistoryTracker =
   new FirestoreBigQueryEventHistoryTracker(eventTrackerConfig);
 
 // Initialize logging.
+logs.logger.setLogLevel(config.logLevel);
 logs.init();
 
 /** Initialize Firebase Admin SDK if not already initialized */
@@ -212,10 +214,7 @@ export const fsexportbigquery = functions.firestore
         context
       );
     } catch (err) {
-      functions.logger.warn(
-        "Failed to write event to BigQuery Immediately. Will attempt to Enqueue to Cloud Tasks.",
-        err
-      );
+      logs.failedToWriteToBigQueryImmediately(err as Error);
       // Handle enqueue errors with retries and backup to GCS.
       await attemptToEnqueue(
         err,

firestore-bigquery-export/functions/src/index.ts

@@ -13,131 +13,137 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-import { logger } from "firebase-functions";
 import config from "./config";
-import { ChangeType } from "@firebaseextensions/firestore-bigquery-change-tracker";
+import {
+  ChangeType,
+  Logger,
+} from "@firebaseextensions/firestore-bigquery-change-tracker";
+
+export const logger = new Logger();
 
 export const arrayFieldInvalid = (fieldName: string) => {
   logger.warn(`Array field '${fieldName}' does not contain an array, skipping`);
 };
 
 export const bigQueryDatasetCreated = (datasetId: string) => {
-  logger.log(`Created BigQuery dataset: ${datasetId}`);
+  logger.info(`Created BigQuery dataset: ${datasetId}`);
 };
 
 export const bigQueryDatasetCreating = (datasetId: string) => {
-  logger.log(`Creating BigQuery dataset: ${datasetId}`);
+  logger.debug(`Creating BigQuery dataset: ${datasetId}`);
 };
 
 export const bigQueryDatasetExists = (datasetId: string) => {
-  logger.log(`BigQuery dataset already exists: ${datasetId}`);
+  logger.info(`BigQuery dataset already exists: ${datasetId}`);
 };
 
 export const bigQueryErrorRecordingDocumentChange = (e: Error) => {
   logger.error(`Error recording document changes.`, e);
 };
 
 export const bigQueryLatestSnapshotViewQueryCreated = (query: string) => {
-  logger.log(`BigQuery latest snapshot view query:\n${query}`);
+  logger.info(`BigQuery latest snapshot view query:\n${query}`);
 };
 
 export const bigQueryTableAlreadyExists = (
   tableName: string,
   datasetName: string
 ) => {
-  logger.log(
+  logger.info(
     `BigQuery table with name ${tableName} already ` +
       `exists in dataset ${datasetName}!`
   );
 };
 
 export const bigQueryTableCreated = (tableName: string) => {
-  logger.log(`Created BigQuery table: ${tableName}`);
+  logger.info(`Created BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableCreating = (tableName: string) => {
-  logger.log(`Creating BigQuery table: ${tableName}`);
+  logger.debug(`Creating BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableUpdated = (tableName: string) => {
-  logger.log(`Updated existing BigQuery table: ${tableName}`);
+  logger.info(`Updated existing BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableUpdating = (tableName: string) => {
-  logger.log(`Updating existing BigQuery table: ${tableName}`);
+  logger.debug(`Updating existing BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableUpToDate = (tableName: string) => {
-  logger.log(`BigQuery table: ${tableName} is up to date`);
+  logger.info(`BigQuery table: ${tableName} is up to date`);
 };
 
 export const bigQueryTableValidated = (tableName: string) => {
-  logger.log(`Validated existing BigQuery table: ${tableName}`);
+  logger.info(`Validated existing BigQuery table: ${tableName}`);
 };
 
 export const bigQueryTableValidating = (tableName: string) => {
-  logger.log(`Validating existing BigQuery table: ${tableName}`);
+  logger.debug(`Validating existing BigQuery table: ${tableName}`);
 };
 
 export const bigQueryUserDefinedFunctionCreating = (
   functionDefinition: string
 ) => {
-  logger.log(`Creating BigQuery User-defined Function:\n${functionDefinition}`);
+  logger.debug(
+    `Creating BigQuery User-defined Function:\n${functionDefinition}`
+  );
 };
 
 export const bigQueryUserDefinedFunctionCreated = (
   functionDefinition: string
 ) => {
-  logger.log(`Created BigQuery User-defined Function:\n${functionDefinition}`);
+  logger.info(`Created BigQuery User-defined Function:\n${functionDefinition}`);
 };
 
 export const bigQueryViewCreated = (viewName: string) => {
-  logger.log(`Created BigQuery view: ${viewName}`);
+  logger.info(`Created BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewCreating = (viewName: string) => {
-  logger.log(`Creating BigQuery view: ${viewName}`);
+  logger.debug(`Creating BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewAlreadyExists = (
   viewName: string,
   datasetName: string
 ) => {
-  logger.log(
+  logger.info(
     `View with id ${viewName} already exists in dataset ${datasetName}.`
   );
 };
 
 export const bigQueryViewUpdated = (viewName: string) => {
-  logger.log(`Updated existing BigQuery view: ${viewName}`);
+  logger.info(`Updated existing BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewUpdating = (viewName: string) => {
-  logger.log(`Updating existing BigQuery view: ${viewName}`);
+  logger.debug(`Updating existing BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewUpToDate = (viewName: string) => {
-  logger.log(`BigQuery view: ${viewName} is up to date`);
+  logger.info(`BigQuery view: ${viewName} is up to date`);
 };
 
 export const bigQueryViewValidated = (viewName: string) => {
-  logger.log(`Validated existing BigQuery view: ${viewName}`);
+  logger.info(`Validated existing BigQuery view: ${viewName}`);
 };
 
 export const bigQueryViewValidating = (viewName: string) => {
-  logger.log(`Validating existing BigQuery view: ${viewName}`);
+  logger.debug(`Validating existing BigQuery view: ${viewName}`);
 };
 
 export const complete = () => {
-  logger.log("Completed execution of extension");
+  logger.info("Completed execution of extension");
 };
 
 export const dataInserted = (rowCount: number) => {
-  logger.log(`Inserted ${rowCount} row(s) of data into BigQuery`);
+  logger.debug(`Inserted ${rowCount} row(s) of data into BigQuery`);
 };
 
 export const dataInserting = (rowCount: number) => {
-  logger.log(`Inserting ${rowCount} row(s) of data into BigQuery`);
+  logger.debug(`Inserting ${rowCount} row(s) of data into BigQuery`);
 };
 
 export const dataTypeInvalid = (
@@ -171,11 +177,11 @@ export const error = (
 };
 
 export const init = () => {
-  logger.log("Initializing extension with configuration", config);
+  logger.info("Initializing extension with configuration", config);
 };
 
 export const start = () => {
-  logger.log("Started execution of extension with configuration", config);
+  logger.info("Started execution of extension with configuration", config);
 };
 
 export const timestampMissingValue = (fieldName: string) => {
@@ -218,3 +224,10 @@ export const logFailedEventAction = (
     error,
   });
 };
+
+export const failedToWriteToBigQueryImmediately = (error: Error) => {
+  logger.warn(
+    "Failed to write event to BigQuery Immediately. Will attempt to Enqueue to Cloud Tasks.",
+    error
+  );
+};