Merge pull request #412 from AikidoSec/api-discovery-string-formats

Api discovery: Detect string formats

library/agent/Routes.test.ts

@@ -597,3 +597,40 @@ t.test("it ignores body of graphql queries", async (t) => {
     },
   ]);
 });
+
+t.test("with string format", async (t) => {
+  const routes = new Routes(200);
+  routes.addRoute(
+    getContext(
+      "POST",
+      "/body",
+      { "content-type": "application/json" },
+      { email: "[email protected]" }
+    )
+  );
+
+  t.same(routes.asArray(), [
+    {
+      method: "POST",
+      path: "/body",
+      hits: 1,
+      graphql: undefined,
+      apispec: {
+        body: {
+          type: "json",
+          schema: {
+            type: "object",
+            properties: {
+              email: {
+                type: "string",
+                format: "email",
+              },
+            },
+          },
+        },
+        query: undefined,
+        auth: undefined,
+      },
+    },
+  ]);
+});

library/agent/api-discovery/getDataSchema.test.ts

@@ -99,6 +99,36 @@ t.test("it works", async (t) => {
       },
     }
   );
+
+  t.same(
+    getDataSchema({
+      e: "[email protected]",
+      i: "127.0.0.1",
+      u: "http://example.com",
+      d: "2024-10-14",
+    }),
+    {
+      type: "object",
+      properties: {
+        e: {
+          type: "string",
+          format: "email",
+        },
+        i: {
+          type: "string",
+          format: "ipv4",
+        },
+        u: {
+          type: "string",
+          format: "uri",
+        },
+        d: {
+          type: "string",
+          format: "date",
+        },
+      },
+    }
+  );
 });
 
 t.test("test max depth", async (t) => {

library/agent/api-discovery/getDataSchema.ts

@@ -1,3 +1,5 @@
+import { getStringFormat, type StringFormat } from "./getStringFormat";
+
 export type DataSchema = {
   /**
    * Type of this property.
@@ -16,6 +18,10 @@ export type DataSchema = {
    * Data schema for the items of an array.
    */
   items?: DataSchema;
+  /**
+   * Format of the string, if it is a string.
+   */
+  format?: StringFormat;
 };
 
 // Maximum depth to traverse the data structure to get the schema for improved performance
@@ -29,6 +35,12 @@ const maxProperties = 100;
 export function getDataSchema(data: unknown, depth = 0): DataSchema {
   // If the data is not an object (or an array), return the type
   if (typeof data !== "object") {
+    if (typeof data === "string") {
+      const format = getStringFormat(data);
+      if (format) {
+        return { type: "string", format };
+      }
+    }
     return { type: typeof data };
   }
 

library/agent/api-discovery/getStringFormat.test.ts

@@ -0,0 +1,49 @@
+import * as t from "tap";
+import { getStringFormat } from "./getStringFormat";
+
+t.test("it is not a known format", async (t) => {
+  t.same(getStringFormat(""), undefined);
+  t.same(getStringFormat("abc"), undefined);
+  t.same(getStringFormat("2021-11-25T"), undefined);
+  t.same(getStringFormat("2021-11-25T00:00:00"), undefined);
+  t.same(getStringFormat("test".repeat(64)), undefined);
+});
+
+t.test("it is a date string", async (t) => {
+  t.same(getStringFormat("2021-01-01"), "date");
+  t.same(getStringFormat("2021-12-31"), "date");
+});
+
+t.test("it is a date time string", async (t) => {
+  t.same(getStringFormat("1985-04-12T23:20:50.52Z"), "date-time");
+  t.same(getStringFormat("1996-12-19T16:39:57-08:00"), "date-time");
+  t.same(getStringFormat("1990-12-31T23:59:60Z"), "date-time");
+  t.same(getStringFormat("1990-12-31T15:59:60-08:00"), "date-time");
+  t.same(getStringFormat("1937-01-01T12:00:27.87+00:20"), "date-time");
+});
+
+t.test("it is a UUID string", async (t) => {
+  t.same(getStringFormat("550e8400-e29b-41d4-a716-446655440000"), "uuid");
+  t.same(getStringFormat("00000000-0000-0000-0000-000000000000"), "uuid");
+});
+
+t.test("it is an IPv4 string", async (t) => {
+  t.same(getStringFormat("127.0.0.1"), "ipv4");
+  t.same(getStringFormat("1.2.3.4"), "ipv4");
+});
+
+t.test("it is an IPv6 string", async (t) => {
+  t.same(getStringFormat("2001:0db8:85a3:0000:0000:8a2e:0370:7334"), "ipv6");
+  t.same(getStringFormat("2001:db8:0:0:0:8a2e:370:7334"), "ipv6");
+});
+
+t.test("it is an email string", async (t) => {
+  t.same(getStringFormat("[email protected]"), "email");
+  t.same(getStringFormat("ö@ö.de"), "email");
+});
+
+t.test("it is a URI string", async (t) => {
+  t.same(getStringFormat("http://example.com"), "uri");
+  t.same(getStringFormat("https://example.com"), "uri");
+  t.same(getStringFormat("ftp://example.com"), "uri");
+});

library/agent/api-discovery/getStringFormat.ts

@@ -0,0 +1,96 @@
+import { isIPv4, isIPv6 } from "net";
+import isDateString from "./helpers/isDateString";
+import isDateTimeString from "./helpers/isDateTimeString";
+import isUUIDString from "./helpers/isUUIDString";
+import isEmailString from "./helpers/isEmail";
+import isUriString from "./helpers/isUri";
+
+/**
+ * https://swagger.io/docs/specification/v3_0/data-models/data-types/#strings
+ */
+export type StringFormat =
+  | "date"
+  | "date-time"
+  | "email"
+  | "uuid"
+  | "uri"
+  | "ipv4"
+  | "ipv6";
+
+// Used for improved performance
+const indicationChars = new Set<string>(["-", ":", "@", ".", "://"]);
+
+/**
+ * Get the format of a string
+ * https://swagger.io/docs/specification/v3_0/data-models/data-types/#strings
+ */
+export function getStringFormat(str: string): StringFormat | undefined {
+  // Skip if too short
+  if (str.length < 5) {
+    return undefined;
+  }
+
+  // Skip if too long (performance optimization)
+  if (str.length > 255) {
+    return undefined;
+  }
+
+  const foundIndicationChars = checkForIndicationChars(str);
+
+  if (foundIndicationChars.has("-")) {
+    if (foundIndicationChars.has(":")) {
+      // Check if it is a date-time, e.g. 2021-01-01T00:00:00Z
+      if (isDateTimeString(str)) {
+        return "date-time";
+      }
+    }
+
+    // Check if it is a date, e.g. 2021-01-01
+    if (isDateString(str)) {
+      return "date";
+    }
+
+    // Check if it is a UUID
+    if (isUUIDString(str)) {
+      return "uuid";
+    }
+  }
+
+  // Check if it is an email
+  if (foundIndicationChars.has("@") && isEmailString(str)) {
+    return "email";
+  }
+
+  // Check if it is a URI
+  if (foundIndicationChars.has("://") && isUriString(str)) {
+    return "uri";
+  }
+
+  // Check if it is an IPv4
+  if (foundIndicationChars.has(".") && isIPv4(str)) {
+    return "ipv4";
+  }
+
+  // Check if it is an IPv6
+  if (foundIndicationChars.has(":") && isIPv6(str)) {
+    return "ipv6";
+  }
+
+  return undefined;
+}
+
+/**
+ * Check for indication characters in a string
+ * This is used to improve performance
+ */
+function checkForIndicationChars(str: string): Set<string> {
+  const foundChars = new Set<string>();
+
+  for (const iChar of indicationChars) {
+    if (str.includes(iChar)) {
+      foundChars.add(iChar);
+    }
+  }
+
+  return foundChars;
+}

library/agent/api-discovery/helpers/isDateString.test.ts

@@ -0,0 +1,16 @@
+import * as t from "tap";
+import isDateString from "./isDateString";
+
+t.test("it is a date string", async (t) => {
+  t.same(isDateString("2021-01-01"), true);
+  t.same(isDateString("2021-12-31"), true);
+});
+
+t.test("it is not a date string", async (t) => {
+  t.same(isDateString("2021-01-32"), false);
+  t.same(isDateString("-2021-01-20"), false);
+  t.same(isDateString(""), false);
+  t.same(isDateString("2021-01-01T00:00:00Z"), false);
+  t.same(isDateString("01"), false);
+  t.same(isDateString("2001-ab-02"), false);
+});

library/agent/api-discovery/helpers/isDateString.ts

@@ -0,0 +1,21 @@
+/**
+ * Checks if the string is a date according to RFC3339
+ * https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
+ */
+export default function isDateString(str: string): boolean {
+  if (str.length !== 10) {
+    return false;
+  }
+
+  if (!/^\d{4}-\d{2}-\d{2}$/.test(str)) {
+    return false;
+  }
+
+  const [year, month, day] = str.split("-").map(Number);
+
+  if (month < 1 || month > 12 || day < 1 || day > 31 || year < 0) {
+    return false;
+  }
+
+  return true;
+}

library/agent/api-discovery/helpers/isDateTimeString.test.ts

@@ -0,0 +1,22 @@
+import * as t from "tap";
+import isDateTimeString from "./isDateTimeString";
+
+t.test("it is a date time string", async (t) => {
+  t.same(isDateTimeString("1985-04-12T23:20:50.52Z"), true);
+  t.same(isDateTimeString("1996-12-19T16:39:57-08:00"), true);
+  t.same(isDateTimeString("1990-12-31T23:59:60Z"), true);
+  t.same(isDateTimeString("1990-12-31T15:59:60-08:00"), true);
+  t.same(isDateTimeString("1937-01-01T12:00:27.87+00:20"), true);
+});
+
+t.test("it is not a date time string", async (t) => {
+  t.same(isDateTimeString(""), false);
+  t.same(isDateTimeString("2021-11-25"), false);
+  t.same(isDateTimeString("2021-11-25T"), false);
+  t.same(isDateTimeString("2021-11-25T00:00:00"), false);
+  t.same(isDateTimeString("2021-13-05T00:00:00+00:00"), false);
+  t.same(isDateTimeString("2021-999-05T00:00:00+00:00"), false);
+  t.same(isDateTimeString("2021-02-05T00:90:00+00:00"), false);
+  t.same(isDateTimeString("2021-02-05T00:00:00+90:00"), false);
+  t.same(isDateTimeString("2021-02-05T00:00:00+00:000000000"), false);
+});

library/agent/api-discovery/helpers/isDateTimeString.ts

@@ -0,0 +1,53 @@
+const pattern =
+  /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/i;
+
+/**
+ * Checks if the string is a date time according to RFC3339
+ * https://datatracker.ietf.org/doc/html/rfc3339#section-5.6
+ */
+export default function isDateTimeString(str: string): boolean {
+  if (str.length < 20 || str.length > 29) {
+    return false;
+  }
+
+  if (!pattern.test(str)) {
+    return false;
+  }
+
+  const [date, time] = str.split("T");
+
+  // Validate date value
+  const [year, month, day] = date.split("-").map(Number);
+  if (month < 1 || month > 12 || day < 1 || day > 31 || year < 0) {
+    return false;
+  }
+
+  // Validate time value
+  const [hour, minute, second] = time.split(":").map(Number);
+  if (
+    hour < 0 ||
+    hour > 23 ||
+    minute < 0 ||
+    minute > 59 ||
+    second < 0 ||
+    second > 59
+  ) {
+    return false;
+  }
+
+  // Validate time offset value
+  if (!str.endsWith("Z")) {
+    const offset = str.slice(-5);
+    const [offsetHour, offsetMinute] = offset.split(":").map(Number);
+    if (
+      offsetHour < 0 ||
+      offsetHour > 23 ||
+      offsetMinute < 0 ||
+      offsetMinute > 59
+    ) {
+      return false;
+    }
+  }
+
+  return true;
+}