Allow arbitrary JSON in google.protobuf.Struct fields during validation

common/protos/BUILD

@@ -1,19 +1,42 @@
 package(default_visibility = ["//visibility:public"])
 
 load("//tools:ts_library.bzl", "ts_library")
+load("@df//testing:index.bzl", "ts_test_suite")
 
 ts_library(
     name = "protos",
     srcs = glob(
         ["*.ts"],
+        exclude = ["*_test.ts", "*.spec.ts"],
     ),
     deps = [
         "//:modules-fix",
         "//common/strings",
+        "//protos:ts",
         "@npm//protobufjs",
     ],
 )
 
+ts_test_suite(
+    name = "tests",
+    srcs = glob(["*_test.ts"]),
+    data = [
+        "@npm//source-map-support",
+    ],
+    templated_args = [
+        "--node_options=--require=source-map-support/register",
+        "--bazel_patch_module_resolver",
+    ],
+    deps = [
+        ":protos",
+        "//protos:ts",
+        "@df//testing",
+        "@npm//@types/chai",
+        "@npm//@types/node",
+        "@npm//chai",
+    ],
+)
+
 load("//testing:build_test.bzl", "build_test")
 
 build_test(

common/protos/index.ts

@@ -1,5 +1,7 @@
 import { util } from "protobufjs";
 
+import { google } from "df/protos/ts";
+
 const CONFIGS_PROTO_DOCUMENTATION_URL =
   "https://dataform-co.github.io/dataform/docs/configs-reference";
 const REPORT_ISSUE_URL = "https://github.com/dataform-co/dataform/issues";
@@ -15,6 +17,8 @@ export interface IProtoClass<IProto, Proto> {
   toObject(proto: Proto): { [k: string]: any };
   fromObject(obj: { [k: string]: any }): Proto;
 
+  verify(obj: any): (string | null);
+
   getTypeUrl(prefix: string): string;
 }
 
@@ -24,6 +28,24 @@ export enum VerifyProtoErrorBehaviour {
   SHOW_DOCS_LINK
 }
 
+const Struct = google.protobuf.Struct;
+
+// Save references to the original generated methods
+const originalVerify = Struct.verify;
+
+// Monkey Patching Methods
+Struct.verify = function (object: any) {
+  if (object && typeof object === "object" && !("fields" in object)) {
+    const fields: { [key: string]: any } = {};
+    for (const [k, v] of Object.entries(object)) {
+      fields[k] = unknownToValueShallow(v);
+    }
+    Object.keys(object).forEach(key => delete object[key]);
+    object.fields = fields;
+  }
+  return originalVerify.call(this, object);
+};
+
 // This is a minimalist Typescript equivalent for the validation part of Profobuf's JsonFormat's
 // mergeMessage method:
 // https://github.com/protocolbuffers/protobuf/blob/670e0c2a0d0b64c994f743a73ee9b8926c47580d/java/util/src/main/java/com/google/protobuf/util/JsonFormat.java#L1455
@@ -42,6 +64,7 @@ export function verifyObjectMatchesProto<Proto>(
   }
 
   // Calling toObject on the object/JSON creates a version only contains the valid proto fields.
+  protoType.verify(object);
   const proto = protoType.create(object);
   const protoCastObject = protoType.toObject(proto);
 
@@ -58,7 +81,7 @@ export function verifyObjectMatchesProto<Proto>(
         if (!presentValue) {
           throw ReferenceError(
             `Unexpected empty value for "${presentKey}".` +
-              maybeGetDocsLinkPrefix(errorBehaviour, protoType)
+            maybeGetDocsLinkPrefix(errorBehaviour, protoType)
           );
         }
         if (typeof presentValue === "object" && Object.keys(presentValue).length === 0) {
@@ -70,13 +93,13 @@ export function verifyObjectMatchesProto<Proto>(
             `Unexpected property "${presentKey}" for "${protoType
               .getTypeUrl("")
               .replace("/", "")}", please report this to the Dataform team at ` +
-              `${REPORT_ISSUE_URL}.`
+            `${REPORT_ISSUE_URL}.`
           );
         }
         throw ReferenceError(
           `Unexpected property "${presentKey}", or property value type of ` +
-            `"${typeof presentValue}" is incorrect.` +
-            maybeGetDocsLinkPrefix(errorBehaviour, protoType)
+          `"${typeof presentValue}" is incorrect.` +
+          maybeGetDocsLinkPrefix(errorBehaviour, protoType)
         );
       }
       if (typeof presentValue === "object") {
@@ -140,3 +163,53 @@ function fromBase64(value: string): Uint8Array {
   util.base64.decode(value, buf, 0);
   return buf;
 }
+
+function unknownToValueShallow(raw: unknown): google.protobuf.IValue {
+  if (raw === null || typeof raw === "undefined") {
+    return { nullValue: 0 };
+  }
+  if (typeof raw === "string") {
+    return { stringValue: raw };
+  }
+  if (typeof raw === "number") {
+    return { numberValue: raw };
+  }
+  if (typeof raw === "boolean") {
+    return { boolValue: raw };
+  }
+  if (Array.isArray(raw)) {
+    return { listValue: { values: raw.map(unknownToValueShallow) } };
+  }
+  if (typeof raw === "object") {
+    return { structValue: raw as any };
+  }
+  throw new Error(`Unsupported value: ${raw}`);
+}
+
+export function unknownToValue(raw: unknown): google.protobuf.IValue {
+  if (raw === null || typeof raw === "undefined") {
+    return { nullValue: 0 };
+  }
+  if (typeof raw === "string") {
+    return { stringValue: raw };
+  }
+  if (typeof raw === "number") {
+    return { numberValue: raw };
+  }
+  if (typeof raw === "boolean") {
+    return { boolValue: raw };
+  }
+  if (Array.isArray(raw)) {
+    return { listValue: { values: raw.map(unknownToValue) } };
+  }
+  if (typeof raw === "object") {
+    return {
+      structValue: {
+        fields: Object.fromEntries(
+          Object.entries(raw).map(([key, value]) => [key, unknownToValue(value)])
+        )
+      }
+    };
+  }
+  throw new Error(`Unsupported value: ${raw}`);
+}

common/protos/index_test.ts

@@ -0,0 +1,45 @@
+import { expect } from "chai";
+import { verifyObjectMatchesProto, VerifyProtoErrorBehaviour } from "./index";
+
+import { dataform } from "df/protos/ts";
+import { suite, test } from "df/testing";
+
+suite("verifyObjectMatchesProto", () => {
+  test("throws error when top-level object is an array", () => {
+    expect(() => {
+      verifyObjectMatchesProto(dataform.Target, [] as any);
+    }).to.throw(ReferenceError, "Expected a top-level object, but found an array");
+  });
+
+  test("throws error when null value provided for array field and SHOW_DOCS_LINK", () => {
+    expect(() => {
+      verifyObjectMatchesProto(
+        dataform.Table,
+        { dependencyTargets: null } as any,
+        VerifyProtoErrorBehaviour.SHOW_DOCS_LINK
+      );
+    }).to.throw(ReferenceError, /Unexpected empty value for "dependencyTargets"/);
+  });
+
+  test("throws error on type mismatch with SUGGEST_REPORTING_TO_DATAFORM_TEAM", () => {
+    expect(() => {
+      verifyObjectMatchesProto(
+        dataform.Table,
+        { actionDescriptor: 123 } as any,
+        VerifyProtoErrorBehaviour.SUGGEST_REPORTING_TO_DATAFORM_TEAM
+      );
+    }).to.throw(
+      ReferenceError,
+      /Unexpected property "actionDescriptor" for ".*Table".*please report this to the Dataform team/
+    );
+  });
+
+  test("throws error on default type mismatch", () => {
+    expect(() => {
+      verifyObjectMatchesProto(dataform.Table, { actionDescriptor: 123 } as any);
+    }).to.throw(
+      ReferenceError,
+      /Unexpected property "actionDescriptor", or property value type of "number" is incorrect/
+    );
+  });
+});