feat(util-dynamodb): enable undefined values removal in marshall

packages/util-dynamodb/src/convertToAttr.spec.ts

@@ -1,6 +1,7 @@
 import { AttributeValue } from "@aws-sdk/client-dynamodb";
 
 import { convertToAttr } from "./convertToAttr";
+import { marshallOptions } from "./marshall";
 import { NativeAttributeValue } from "./models";
 
 describe("convertToAttr", () => {
@@ -179,6 +180,31 @@ describe("convertToAttr", () => {
         L: [{ NULL: true }, { NULL: true }, { NULL: true }],
       });
     });
+
+    describe(`testing list with options.removeUndefinedValues`, () => {
+      describe("throws error", () => {
+        const testErrorListWithUndefinedValues = (options?: marshallOptions) => {
+          expect(() => {
+            convertToAttr(["defined", undefined], options);
+          }).toThrowError(`Pass options.removeUndefinedValues=true to remove undefined values from map/array/set.`);
+        };
+
+        [undefined, {}, { convertEmptyValues: false }].forEach((options) => {
+          it(`when options=${options}`, () => {
+            testErrorListWithUndefinedValues(options);
+          });
+        });
+      });
+
+      it(`returns when options.removeUndefinedValues=true`, () => {
+        expect(convertToAttr(["defined", undefined], { removeUndefinedValues: true })).toEqual({
+          L: [{ S: "defined" }],
+        });
+        expect(convertToAttr([undefined, "defined", undefined], { removeUndefinedValues: true })).toEqual({
+          L: [{ S: "defined" }],
+        });
+      });
+    });
   });
 
   describe("set", () => {
@@ -204,21 +230,53 @@ describe("convertToAttr", () => {
       expect(convertToAttr(set)).toEqual({ SS: Array.from(set) });
     });
 
-    it("returns null for empty set for options.convertEmptyValues=true", () => {
-      expect(convertToAttr(new Set([]), { convertEmptyValues: true })).toEqual({ NULL: true });
+    describe("set with undefined", () => {
+      describe("throws error", () => {
+        const testErrorSetWithUndefined = (options?: marshallOptions) => {
+          expect(() => {
+            convertToAttr(new Set([1, undefined, 3]), options);
+          }).toThrowError(`Pass options.removeUndefinedValues=true to remove undefined values from map/array/set.`);
+        };
+
+        [undefined, {}, { convertEmptyValues: false }].forEach((options) => {
+          it(`when options=${options}`, () => {
+            testErrorSetWithUndefined(options);
+          });
+        });
+      });
+
+      it("returns when options.removeUndefinedValues=true", () => {
+        expect(convertToAttr(new Set([1, undefined, 3]), { removeUndefinedValues: true })).toEqual({ NS: ["1", "3"] });
+      });
     });
 
-    it("throws error for empty set", () => {
-      expect(() => {
-        convertToAttr(new Set([]));
-      }).toThrowError(`Please pass a non-empty set, or set convertEmptyValues to true.`);
+    describe("empty set", () => {
+      describe("throws error", () => {
+        const testErrorEmptySet = (options?: marshallOptions) => {
+          expect(() => {
+            convertToAttr(new Set([]), options);
+          }).toThrowError(`Pass a non-empty set, or options.convertEmptyValues=true.`);
+        };
+
+        [undefined, {}, { convertEmptyValues: false }].forEach((options) => {
+          it(`when options=${options}`, () => {
+            testErrorEmptySet(options);
+          });
+        });
+      });
+
+      it("returns null when options.convertEmptyValues=true", () => {
+        expect(convertToAttr(new Set([]), { convertEmptyValues: true })).toEqual({ NULL: true });
+      });
     });
 
-    it("thows error for unallowed set", () => {
-      expect(() => {
-        // @ts-expect-error Type 'Set<boolean>' is not assignable
-        convertToAttr(new Set([true, false]));
-      }).toThrowError(`Only Number Set (NS), Binary Set (BS) or String Set (SS) are allowed.`);
+    describe("unallowed set", () => {
+      it("throws error", () => {
+        expect(() => {
+          // @ts-expect-error Type 'Set<boolean>' is not assignable
+          convertToAttr(new Set([true, false]));
+        }).toThrowError(`Only Number Set (NS), Binary Set (BS) or String Set (SS) are allowed.`);
+      });
     });
   });
 
@@ -278,6 +336,29 @@ describe("convertToAttr", () => {
         M: { stringKey: { NULL: true }, binaryKey: { NULL: true }, setKey: { NULL: true } },
       });
     });
+
+    describe(`testing map with options.removeUndefinedValues`, () => {
+      describe("throws error", () => {
+        const testErrorMapWithUndefinedValues = (options?: marshallOptions) => {
+          expect(() => {
+            convertToAttr({ definedKey: "definedKey", undefinedKey: undefined }, options);
+          }).toThrowError(`Pass options.removeUndefinedValues=true to remove undefined values from map/array/set.`);
+        };
+
+        [undefined, {}, { convertEmptyValues: false }].forEach((options) => {
+          it(`when options=${options}`, () => {
+            testErrorMapWithUndefinedValues(options);
+          });
+        });
+      });
+
+      it(`returns when options.removeUndefinedValues=true`, () => {
+        const input = { definedKey: "definedKey", undefinedKey: undefined };
+        expect(convertToAttr(input, { removeUndefinedValues: true })).toEqual({
+          M: { definedKey: { S: "definedKey" } },
+        });
+      });
+    });
   });
 
   describe("string", () => {
@@ -297,8 +378,14 @@ describe("convertToAttr", () => {
       constructor(private readonly foo: string) {}
     }
 
+    it(`throws for: undefined`, () => {
+      expect(() => {
+        convertToAttr(undefined);
+      }).toThrowError(`Pass options.removeUndefinedValues=true to remove undefined values from map/array/set.`);
+    });
+
     // ToDo: Serialize ES6 class objects as string https://github.com/aws/aws-sdk-js-v3/issues/1535
-    [undefined, new Date(), new FooObj("foo")].forEach((data) => {
+    [new Date(), new FooObj("foo")].forEach((data) => {
       it(`throws for: ${String(data)}`, () => {
         expect(() => {
           // @ts-expect-error Argument is not assignable to parameter of type 'NativeAttributeValue'

packages/util-dynamodb/src/convertToAttr.ts

@@ -22,44 +22,52 @@ export const convertToAttr = (data: NativeAttributeValue, options?: marshallOpti
 };
 
 const convertToListAttr = (data: NativeAttributeValue[], options?: marshallOptions): { L: AttributeValue[] } => ({
-  L: data.map((item) => convertToAttr(item, options)),
+  L: data
+    .filter((item) => !options?.removeUndefinedValues || (options?.removeUndefinedValues && item !== undefined))
+    .map((item) => convertToAttr(item, options)),
 });
 
 const convertToSetAttr = (
   set: Set<any>,
   options?: marshallOptions
 ): { NS: string[] } | { BS: Uint8Array[] } | { SS: string[] } | { NULL: true } => {
-  if (set.size === 0) {
+  const setToOperate = options?.removeUndefinedValues ? new Set([...set].filter((value) => value !== undefined)) : set;
+
+  if (!options?.removeUndefinedValues && setToOperate.has(undefined)) {
+    throw new Error(`Pass options.removeUndefinedValues=true to remove undefined values from map/array/set.`);
+  }
+
+  if (setToOperate.size === 0) {
     if (options?.convertEmptyValues) {
       return convertToNullAttr();
     }
-    throw new Error(`Please pass a non-empty set, or set convertEmptyValues to true.`);
+    throw new Error(`Pass a non-empty set, or options.convertEmptyValues=true.`);
   }
 
-  const item = set.values().next().value;
+  const item = setToOperate.values().next().value;
   if (typeof item === "number") {
     return {
-      NS: Array.from(set)
+      NS: Array.from(setToOperate)
         .map(convertToNumberAttr)
         .map((item) => item.N),
     };
   } else if (typeof item === "bigint") {
     return {
-      NS: Array.from(set)
+      NS: Array.from(setToOperate)
         .map(convertToBigIntAttr)
         .map((item) => item.N),
     };
   } else if (typeof item === "string") {
     return {
-      SS: Array.from(set)
+      SS: Array.from(setToOperate)
         .map(convertToStringAttr)
         .map((item) => item.S),
     };
   } else if (isBinary(item)) {
     return {
       // Do not alter binary data passed https://github.com/aws/aws-sdk-js-v3/issues/1530
       // @ts-expect-error Type 'ArrayBuffer' is not assignable to type 'Uint8Array'
-      BS: Array.from(set)
+      BS: Array.from(setToOperate)
         .map(convertToBinaryAttr)
         .map((item) => item.B),
     };
@@ -72,17 +80,24 @@ const convertToMapAttr = (
   data: { [key: string]: NativeAttributeValue },
   options?: marshallOptions
 ): { M: { [key: string]: AttributeValue } } => ({
-  M: Object.entries(data).reduce(
-    (acc: { [key: string]: AttributeValue }, [key, value]: [string, NativeAttributeValue]) => ({
-      ...acc,
-      [key]: convertToAttr(value, options),
-    }),
-    {}
-  ),
+  M: Object.entries(data)
+    .filter(
+      ([key, value]: [string, NativeAttributeValue]) =>
+        !options?.removeUndefinedValues || (options?.removeUndefinedValues && value !== undefined)
+    )
+    .reduce(
+      (acc: { [key: string]: AttributeValue }, [key, value]: [string, NativeAttributeValue]) => ({
+        ...acc,
+        [key]: convertToAttr(value, options),
+      }),
+      {}
+    ),
 });
 
 const convertToScalarAttr = (data: NativeScalarAttributeValue, options?: marshallOptions): AttributeValue => {
-  if (data === null && typeof data === "object") {
+  if (data === undefined) {
+    throw new Error(`Pass options.removeUndefinedValues=true to remove undefined values from map/array/set.`);
+  } else if (data === null && typeof data === "object") {
     return convertToNullAttr();
   } else if (typeof data === "boolean") {
     return { BOOL: data };

packages/util-dynamodb/src/marshall.spec.ts

@@ -15,18 +15,20 @@ describe("marshall", () => {
   });
 
   it("calls convertToAttr", () => {
-    // @ts-ignore output mocked for testing
     expect(marshall(input)).toEqual(input);
     expect(convertToAttr).toHaveBeenCalledTimes(1);
     expect(convertToAttr).toHaveBeenCalledWith(input, undefined);
   });
 
-  [false, true].forEach((convertEmptyValues) => {
-    it(`passes convertEmptyValues=${convertEmptyValues} to convertToAttr`, () => {
-      // @ts-ignore output mocked for testing
-      expect(marshall(input, { convertEmptyValues })).toEqual(input);
-      expect(convertToAttr).toHaveBeenCalledTimes(1);
-      expect(convertToAttr).toHaveBeenCalledWith(input, { convertEmptyValues });
+  ["convertEmptyValues", "removeUndefinedValues"].forEach((option) => {
+    describe(`options.${option}`, () => {
+      [false, true].forEach((value) => {
+        it(`passes ${value} to convertToAttr`, () => {
+          expect(marshall(input, { [option]: value })).toEqual(input);
+          expect(convertToAttr).toHaveBeenCalledTimes(1);
+          expect(convertToAttr).toHaveBeenCalledWith(input, { [option]: value });
+        });
+      });
     });
   });
 });

packages/util-dynamodb/src/marshall.ts

@@ -11,6 +11,10 @@ export interface marshallOptions {
    * Whether to automatically convert empty strings, blobs, and sets to `null`
    */
   convertEmptyValues?: boolean;
+  /**
+   * Whether to remove undefined values while marshalling.
+   */
+  removeUndefinedValues?: boolean;
 }
 
 /**

packages/util-dynamodb/src/models.ts

@@ -14,10 +14,11 @@ export type NativeAttributeValue =
   | NativeScalarAttributeValue
   | { [key: string]: NativeAttributeValue }
   | NativeAttributeValue[]
-  | Set<number | bigint | NumberValue | string | NativeAttributeBinary>;
+  | Set<number | bigint | NumberValue | string | NativeAttributeBinary | undefined>;
 
 export type NativeScalarAttributeValue =
   | null
+  | undefined
   | boolean
   | number
   | NumberValue