Unary estimator for detecting names and transforming to gender (#445)

* Added unary estimator for detecting names in text fields and looking up likely gender in government data

* Added custom map return type, NameStats, for said unary estimator

core/src/main/scala/com/salesforce/op/dsl/RichTextFeature.scala

@@ -434,6 +434,15 @@ trait RichTextFeature {
       toLowercase: Boolean = TextTokenizer.ToLowercase
     ): FeatureLike[Binary] =
       f.transformWith(new SubstringTransformer[T, T2]().setToLowercase(toLowercase), f2)
+
+    /**
+     * Check if feature is actual human names, and if so, return related gender information (English language only)
+     *
+     * @param threshold optional, fraction of rows containing names before processing (default = 0.50)
+     * @return NameStats, a custom map that will be empty if no name was found
+     */
+    def identifyIfHumanName(threshold: Double = 0.50): FeatureLike[NameStats] =
+      new HumanNameDetector[T]().setThreshold(threshold).setInput(f).getOutput()
   }
 
   implicit class RichPhoneFeature(val f: FeatureLike[Phone]) {

core/src/main/scala/com/salesforce/op/stages/impl/feature/HumanNameDetector.scala

@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2017, Salesforce.com, Inc.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are met:
+ *
+ * * Redistributions of source code must retain the above copyright notice, this
+ *   list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright notice,
+ *   this list of conditions and the following disclaimer in the documentation
+ *   and/or other materials provided with the distribution.
+ *
+ * * Neither the name of the copyright holder nor the names of its
+ *   contributors may be used to endorse or promote products derived from
+ *   this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+ * CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+package com.salesforce.op.stages.impl.feature
+
+import com.salesforce.op._
+import com.salesforce.op.features.types.NameStats.GenderValue
+import com.salesforce.op.features.types._
+import com.salesforce.op.stages.base.unary.{UnaryEstimator, UnaryModel}
+import com.salesforce.op.stages.impl.MetadataLike
+import com.salesforce.op.utils.spark.RichDataset._
+import com.salesforce.op.utils.stages._
+import com.twitter.algebird.Operators._
+import org.apache.spark.sql._
+import org.apache.spark.sql.types.{Metadata, MetadataBuilder}
+
+import scala.reflect.runtime.universe.TypeTag
+
+/**
+ * Unary estimator for identifying whether a single Text column is a name or not. If the column does appear to be a
+ * name, a custom map will be returned that contains the guessed gender for each entry (gender detection only supported
+ * for English at the moment). If the column does not appear to be a name, then the output will be an empty map.
+ * @param uid           uid for instance
+ * @param operationName unique name of the operation this stage performs
+ * @param tti           type tag for input
+ * @param ttiv          type tag for input value
+ * @tparam T            the FeatureType (subtype of Text) to operate over
+ */
+class HumanNameDetector[T <: Text]
+(
+  uid: String = UID[HumanNameDetector[T]],
+  operationName: String = "humanNameDetect"
+)
+(
+  implicit tti: TypeTag[T],
+  override val ttiv: TypeTag[T#Value]
+) extends UnaryEstimator[T, NameStats](
+  uid = uid,
+  operationName = operationName
+) with NameDetectFun[T] {
+
+  def fitFn(dataset: Dataset[T#Value]): HumanNameDetectorModel[T] = {
+    implicit val (nameDetectStatsEnc, nameDetectStatsMonoid) = (NameDetectStats.kryo, NameDetectStats.monoid)
+    val mapFun: T#Value => NameDetectStats = makeMapFunction(dataset.sparkSession)
+    val aggResults: NameDetectStats = dataset.map(mapFun).reduce(_ + _)
+    val treatAsName = computeTreatAsName(aggResults)
+
+    val newMetadata = HumanNameDetectorMetadata(
+      treatAsName, aggResults.dictCheckResult.value, aggResults.genderResultsByStrategy
+    ).toMetadata()
+    val metaDataBuilder = new MetadataBuilder().withMetadata(getMetadata()).withMetadata(newMetadata)
+    setMetadata(metaDataBuilder.build())
+
+    val orderedGenderDetectStrategies =
+      if (treatAsName) orderGenderStrategies(aggResults) else Seq.empty[GenderDetectStrategy]
+    new HumanNameDetectorModel[T](uid, operationName, treatAsName, orderedGenderDetectStrategies)
+  }
+}
+
+class HumanNameDetectorModel[T <: Text]
+(
+  override val uid: String,
+  operationName: String,
+  val treatAsName: Boolean,
+  val orderedGenderDetectStrategies: Seq[GenderDetectStrategy] = Seq.empty[GenderDetectStrategy]
+)(implicit tti: TypeTag[T])
+  extends UnaryModel[T, NameStats](operationName, uid) with NameDetectFun[T] {
+
+  import NameStats.GenderValue.GenderNA
+  import NameStats.Key._
+  def transformFn: T => NameStats = (input: T) => {
+    val tokens = preProcess(input)
+    if (treatAsName) {
+      require(orderedGenderDetectStrategies.nonEmpty, "There must be a gender extraction strategy if treating as name.")
+      // Could figure out how to use a broadcast variable for the gender dictionary within a unary transformer
+      val genders: Seq[GenderValue] = orderedGenderDetectStrategies map {
+        identifyGender(input.value, tokens, _, NameDetectUtils.DefaultGenderDictionary)
+      }
+      val gender = genders.find(_ != GenderNA).getOrElse(GenderNA)
+      val map: Map[String, String] = Map(
+        IsName.toString -> true.toString,
+        OriginalValue.toString -> input.value.getOrElse(""),
+        Gender.toString -> gender.toString
+      )
+      NameStats(map)
+    }
+    else NameStats(Map.empty[String, String])
+  }
+}
+
+case class HumanNameDetectorMetadata
+(
+  treatAsName: Boolean,
+  predictedNameProb: Double,
+  genderResultsByStrategy: Map[String, GenderStats]
+) extends MetadataLike {
+  import HumanNameDetectorMetadata._
+
+  override def toMetadata(): Metadata = {
+    val metaDataBuilder = new MetadataBuilder()
+    metaDataBuilder.putBoolean(TreatAsNameKey, treatAsName)
+    metaDataBuilder.putDouble(PredictedNameProbKey, predictedNameProb)
+    val genderResultsMetaDataBuilder = new MetadataBuilder()
+    genderResultsByStrategy foreach { case (strategyString, stats) =>
+      genderResultsMetaDataBuilder.putDoubleArray(strategyString, Array(stats.numMale, stats.numFemale, stats.numOther))
+    }
+    metaDataBuilder.putMetadata(GenderResultsByStrategyKey, genderResultsMetaDataBuilder.build())
+    metaDataBuilder.build()
+  }
+
+  override def toMetadata(skipUnsupported: Boolean): Metadata = toMetadata()
+}
+
+case object HumanNameDetectorMetadata {
+  val TreatAsNameKey = "treatAsName"
+  val PredictedNameProbKey = "predictedNameProb"
+  val GenderResultsByStrategyKey = "genderResultsByStrategy"
+
+  def fromMetadata(metadata: Metadata): HumanNameDetectorMetadata = {
+    val genderResultsMetadata = metadata.getMetadata(GenderResultsByStrategyKey)
+    val genderResultsByStrategy: Map[String, GenderStats] = {
+      NameDetectUtils.GenderDetectStrategies map { strategy: GenderDetectStrategy =>
+        val strategyString = strategy.toString
+        val resultsArray = genderResultsMetadata.getDoubleArray(strategyString)
+        require(resultsArray.length == 3,
+          "There must be exactly three values for each gender detection strategy: numMale, numFemale, and numOther.")
+        strategyString -> GenderStats(
+          numMale = resultsArray(0).toInt, numFemale = resultsArray(1).toInt, numOther = resultsArray(2).toInt
+        )
+      } toMap
+    }
+    HumanNameDetectorMetadata(
+      metadata.getBoolean(TreatAsNameKey),
+      metadata.getDouble(PredictedNameProbKey),
+      genderResultsByStrategy
+    )
+  }
+}

core/src/main/scala/com/salesforce/op/stages/impl/feature/Transmogrifier.scala

@@ -231,6 +231,10 @@ private[op] case object Transmogrifier {
           val (f, other) = castAs[StreetMap](g) // TODO make Street specific transformer
           f.vectorize(topK = TopK, minSupport = MinSupport, cleanText = CleanText, cleanKeys = CleanKeys,
             others = other, trackNulls = TrackNulls, maxPctCardinality = MaxPercentCardinality)
+        case t if t =:= weakTypeOf[NameStats] =>
+          val (f, other) = castAs[NameStats](g)
+          f.vectorize(topK = TopK, minSupport = MinSupport, cleanText = CleanText, cleanKeys = CleanKeys,
+            others = other, trackNulls = TrackNulls, maxPctCardinality = MaxPercentCardinality)
         case t if t =:= weakTypeOf[GeolocationMap] =>
           val (f, other) = castAs[GeolocationMap](g)
           f.vectorize(cleanKeys = CleanKeys, others = other, trackNulls = TrackNulls)