Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

refactor: Graduate Report post-processing from experimental #1853

Merged
merged 7 commits into from
Oct 25, 2024
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Empty file.
148 changes: 0 additions & 148 deletions experimental/dp_consistency/README.md

This file was deleted.

This file was deleted.

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,16 @@ MAVEN_COORDINATES = "org.wfanet.measurement.reporting:postprocessing-v2alpha:" +
kt_jvm_library(
name = "postprocessing",
srcs = glob(["*.kt"]),
resources = [
"//src/main/python/wfa/measurement/reporting/postprocessing/tools:post_process_origin_report_pyzip",
],
tags = ["maven_coordinates=" + MAVEN_COORDINATES],
deps = [
"//src/main/proto/wfa/measurement/reporting/postprocessing/v2alpha:report_summary_kt_jvm_proto",
"//src/main/proto/wfa/measurement/reporting/v2alpha:report_kt_jvm_proto",
"@wfa_common_jvm//imports/java/com/google/gson",
"@wfa_common_jvm//src/main/kotlin/org/wfanet/measurement/common",
],
)

maven_export(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,10 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package org.wfanet.measurement.reporting.postprocessing
package org.wfanet.measurement.reporting.postprocessing.v2alpha

import com.google.protobuf.InvalidProtocolBufferException
import com.google.protobuf.util.JsonFormat
import org.wfanet.measurement.reporting.MeasurementDetailKt
import org.wfanet.measurement.reporting.ReportSummary
import org.wfanet.measurement.reporting.measurementDetail
import org.wfanet.measurement.reporting.reportSummary
import org.wfanet.measurement.reporting.v2alpha.Metric
import org.wfanet.measurement.reporting.v2alpha.Report

Expand Down Expand Up @@ -51,6 +47,7 @@ object ReportConversion {
return getReportFromJsonString(reportAsJsonString).toReportSummaries()
}

// TODO(@ple13): Move this function to a separate package that handles the tags.
fun getMeasurementPolicy(tag: String): String {
when {
"measurement_policy=AMI" in tag -> return "ami"
Expand All @@ -60,17 +57,20 @@ object ReportConversion {
}
}

// TODO(@ple13): Move this function to a separate package that handles the tags.
fun getSetOperation(tag: String): String {
val parts = tag.split(", ")
val setOperationPart = parts.find { it.startsWith("set_operation=") }
return setOperationPart?.let { it.substringAfter("set_operation=") }
?: error("Set operation must be specified.")
}

// TODO(@ple13): Move this function to a separate package that handles the tags.
fun isCumulative(tag: String): Boolean {
return tag.contains("cumulative=true")
}

// TODO(@ple13): Move this function to a separate package that handles the tags.
fun getTargets(tag: String): List<String> {
val parts = tag.split(", ")
val targetPart = parts.find { it.startsWith("target=") }
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

package org.wfanet.measurement.reporting.postprocessing
package org.wfanet.measurement.reporting.postprocessing.v2alpha

import com.google.gson.GsonBuilder
import java.io.BufferedReader
Expand All @@ -25,16 +25,18 @@ import java.util.logging.Logger
import kotlin.io.path.name
import org.wfanet.measurement.common.getJarResourcePath
import org.wfanet.measurement.common.toJson
import org.wfanet.measurement.reporting.ReportSummary
import org.wfanet.measurement.reporting.v2alpha.Report
import org.wfanet.measurement.reporting.v2alpha.copy
import org.wfanet.measurement.reporting.v2alpha.report

/** Corrects noisy measurements in a report. */
object ReportPostProcessing {
/**
* An implementation of [ReportProcessor] that takes a serialized [Report] in JSON format and
* returns the a [Report] of which all measurements are consistent.
*/
object ReportProcessorImpl : ReportProcessor {
private val logger: Logger = Logger.getLogger(this::class.java.name)
const private val PYTHON_LIBRARY_RESOURCE_NAME =
"experimental/dp_consistency/src/main/python/wfa/measurement/reporting/postprocess/tools/post_process_origin_report.zip"
"src/main/python/wfa/measurement/reporting/postprocessing/tools/post_process_origin_report.zip"
private val resourcePath: Path =
this::class.java.classLoader.getJarResourcePath(PYTHON_LIBRARY_RESOURCE_NAME)
?: error("$PYTHON_LIBRARY_RESOURCE_NAME not found in JAR")
Expand All @@ -52,7 +54,7 @@ object ReportPostProcessing {
* @param report standard JSON serialization of a Report message.
* @return a corrected report, serialized as a standard JSON string.
*/
fun processReportJson(report: String): String {
override fun processReportJson(report: String): String {
return processReport(ReportConversion.getReportFromJsonString(report)).toJson()
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,9 +14,9 @@

syntax = "proto3";

package wfa.measurement.reporting;
package wfa.measurement.reporting.postprocessing.v2alpha;

option java_package = "org.wfanet.measurement.reporting";
option java_package = "org.wfanet.measurement.reporting.postprocessing.v2alpha";
option java_multiple_files = true;
option java_outer_classname = "ReportSummaryProto";

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,6 @@ py_library(
imports = ["../"],
visibility = ["//visibility:public"],
deps = [
"//experimental/dp_consistency/src/main/python/wfa/measurement/reporting/postprocess/noiseninja",
"//src/main/python/wfa/measurement/reporting/postprocessing/noiseninja",
],
)
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ package(default_visibility = ["//visibility:public"])
filegroup(
name = "post_process_origin_report_pyzip",
srcs = [
"//experimental/dp_consistency/src/main/python/wfa/measurement/reporting/postprocess/tools:post_process_origin_report",
"//src/main/python/wfa/measurement/reporting/postprocessing/tools:post_process_origin_report",
],
output_group = "python_zip_file",
)
Expand All @@ -15,8 +15,8 @@ py_binary(
name = "post_process_origin_report",
srcs = ["post_process_origin_report.py"],
deps = [
"//experimental/dp_consistency/src/main/proto/wfa/measurement/reporting/postprocess:report_summary_py_pb2",
"//experimental/dp_consistency/src/main/python/wfa/measurement/reporting/postprocess/report",
"//src/main/proto/wfa/measurement/reporting/postprocessing/v2alpha:report_summary_py_pb2",
"//src/main/python/wfa/measurement/reporting/postprocessing/report",
requirement("openpyxl"),
requirement("pandas"),
],
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,14 +12,12 @@
# See the License for the specific language governing permissions and
# limitations under the License.

import base64
import json
import math
import pandas as pd
import sys

from experimental.dp_consistency.src.main.proto.wfa.measurement.reporting.postprocess import \
report_summary_pb2
from src.main.proto.wfa.measurement.reporting.postprocessing.v2alpha import report_summary_pb2
from functools import partial
from noiseninja.noised_measurements import Measurement
from report.report import Report, MetricReport
Expand Down
Loading
Loading