From 28d2ac499929a4c09036f16e09f49adb96b244ff Mon Sep 17 00:00:00 2001 From: Steven Date: Mon, 2 Dec 2024 19:58:18 +0000 Subject: [PATCH 01/12] updating with new demographic based iu and meta --- app/br/br.cpp | 9 + openbr/core/eval.cpp | 379 ++++++++++++++++++++++++++++++++++++++++++- openbr/core/eval.h | 5 +- openbr/openbr.cpp | 10 ++ openbr/openbr.h | 2 + 5 files changed, 401 insertions(+), 4 deletions(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index 2ece5717c..dceaedee8 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -123,11 +123,19 @@ class FakeMain : public QRunnable } else { br_eval(parv[0], parv[1], parv[2], atoi(parv[3])); } + } else if (!strcmp(fun, "evalfused")) { + check((parc ==6 || (parc == 8)), "Incorrect parameter count for 'evalfused'."); + if (parc == 6) { + br_eval_fused(parv[0], parv[1], parv[2], parv[3], 0, atof(parv[4]), atof(parv[5])); + } else { + br_eval_fused2(parv[0], parv[1], parv[2], parv[3], parv[4], 0, atof(parv[5]), atof(parv[6]), atof(parv[7])); + } } else if (!strcmp(fun, "plot")) { check(parc >= 2, "Incorrect parameter count for 'plot'."); br_plot(parc-1, parv, parv[parc-1], true); } + // Secondary Tasks else if (!strcmp(fun, "fuse")) { check(parc >= 4, "Insufficient parameter count for 'fuse'."); @@ -278,6 +286,7 @@ class FakeMain : public QRunnable "-enroll ... {output_gallery}\n" "-compare [{output}]\n" "-eval [] [{csv}] [{matches}]\n" + "-evalfused [] [{csv}] [{matches}]\n" "-plot ... {destination}\n" "\n" "==== Other Commands ====\n" diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 3b0e5c2ad..09e140476 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -188,7 +188,115 @@ float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsi return Evaluate(scores, truth, csv, target, query, matches); } -float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QString &target, const QString &query, unsigned int matches) +float EvaluateFused(const QString &simmat, const QString &simmat2, const QString &mask, const File &csv, unsigned int matches, float w1, float w2) +{ + qDebug("Evaluating %s %s%s%s", + qPrintable(simmat), + qPrintable(simmat2), + mask.isEmpty() ? "" : qPrintable(" with " + mask), + csv.name.isEmpty() ? "" : qPrintable(" to " + csv)); + + // Read similarity matrix + QString target, query; + Mat scores; + if (simmat.endsWith(".mtx")) { + scores = BEE::readMatrix(simmat, &target, &query); + } else { + QScopedPointer format(Factory::make(simmat)); + scores = format->read(); + } + + QString target2, query2; + Mat scores2; + if (simmat2.endsWith(".mtx")) { + scores2 = BEE::readMatrix(simmat2, &target2, &query2); + } else { + QScopedPointer format(Factory::make(simmat2)); + scores2 = format->read(); + } + + scores = scores*w1 + scores2*w2; + + // Read mask matrix + Mat truth; + if (mask.isEmpty()) { + // Use the galleries specified in the similarity matrix + if (target.isEmpty()) qFatal("Unspecified target gallery."); + if (query.isEmpty()) qFatal("Unspecified query gallery."); + + truth = constructMatchingMask(scores, TemplateList::fromGallery(target).files(), + TemplateList::fromGallery(query).files()); + } else { + File maskFile(mask); + maskFile.set("rows", scores.rows); + maskFile.set("columns", scores.cols); + QScopedPointer format(Factory::make(maskFile)); + truth = format->read(); + } + + return Evaluate(scores, truth, csv, target, query, matches, target2, ""); +} + +float EvaluateFused2(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask, const File &csv, unsigned int matches, float w1, float w2, float w3) +{ + qDebug("Evaluating %s %s%s%s%s", + qPrintable(simmat), + qPrintable(simmat2), + qPrintable(simmat3), + mask.isEmpty() ? "" : qPrintable(" with " + mask), + csv.name.isEmpty() ? "" : qPrintable(" to " + csv)); + + // Read similarity matrix + QString target, query; + Mat scores; + if (simmat.endsWith(".mtx")) { + scores = BEE::readMatrix(simmat, &target, &query); + } else { + QScopedPointer format(Factory::make(simmat)); + scores = format->read(); + } + + QString target2, query2; + Mat scores2; + if (simmat2.endsWith(".mtx")) { + scores2 = BEE::readMatrix(simmat2, &target2, &query2); + } else { + QScopedPointer format(Factory::make(simmat2)); + scores2 = format->read(); + } + + QString target3, query3; + Mat scores3; + if (simmat3.endsWith(".mtx")) { + scores3 = BEE::readMatrix(simmat3, &target3, &query3); + } else { + QScopedPointer format(Factory::make(simmat3)); + scores3 = format->read(); + } + + scores = scores*w1 + scores2*w2 + scores3*w3; + + // Read mask matrix + Mat truth; + if (mask.isEmpty()) { + // Use the galleries specified in the similarity matrix + if (target.isEmpty()) qFatal("Unspecified target gallery."); + if (query.isEmpty()) qFatal("Unspecified query gallery."); + + truth = constructMatchingMask(scores, TemplateList::fromGallery(target).files(), + TemplateList::fromGallery(query).files()); + } else { + File maskFile(mask); + maskFile.set("rows", scores.rows); + maskFile.set("columns", scores.cols); + QScopedPointer format(Factory::make(maskFile)); + truth = format->read(); + } + + return Evaluate(scores, truth, csv, target, query, matches, target2, target3); +} + +float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QString &target, const QString &query, unsigned int matches, const QString &target2, const QString &target3) { if (target.isEmpty() || query.isEmpty()) matches = 0; if (simmat.size() != mask.size()) @@ -301,7 +409,8 @@ float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QStrin index++; } - if (truePositives > previousTruePositives) { + if ((falsePositives > previousFalsePositives) && + (truePositives > previousTruePositives)) { operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount)); if (EERIndex == 0) { @@ -423,7 +532,21 @@ float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QStrin // Attempt to read template size from enrolled gallery and write to output CSV size_t maxSize(0); - if (target.endsWith(".gal") && QFileInfo(target).exists()) { + size_t maxSize2(0); + size_t maxSize3(0); + + if (target.endsWith(".gal") && QFileInfo(target).exists() && target2.endsWith(".gal") && QFileInfo(target2).exists() && target3.endsWith(".gal") && QFileInfo(target3).exists()) { + foreach (const Template &t, TemplateList::fromGallery(target)) maxSize = max(maxSize, t.bytes()); + foreach (const Template &t, TemplateList::fromGallery(target2)) maxSize2 = max(maxSize2, t.bytes()); + foreach (const Template &t, TemplateList::fromGallery(target3)) maxSize3 = max(maxSize3, t.bytes()); + maxSize = maxSize+maxSize2+maxSize3; + lines.append(QString("TS,,%1").arg(QString::number(maxSize))); + } else if (target.endsWith(".gal") && QFileInfo(target).exists() && target2.endsWith(".gal") && QFileInfo(target2).exists()) { + foreach (const Template &t, TemplateList::fromGallery(target)) maxSize = max(maxSize, t.bytes()); + foreach (const Template &t, TemplateList::fromGallery(target2)) maxSize2 = max(maxSize2, t.bytes()); + maxSize = maxSize+maxSize2; + lines.append(QString("TS,,%1").arg(QString::number(maxSize))); + } else if (target.endsWith(".gal") && QFileInfo(target).exists()) { foreach (const Template &t, TemplateList::fromGallery(target)) maxSize = max(maxSize, t.bytes()); lines.append(QString("TS,,%1").arg(QString::number(maxSize))); } @@ -744,6 +867,256 @@ float InplaceEval(const QString &simmat, const QString &mask, const QString &csv return result; } +float InplaceEvalFused(const QString &simmat, const QString &simmat2, const QString &mask, const QString &csv, const float w1, const float w2) +{ + QFile simmatFile(simmat), simmatFile2(simmat2), maskFile(mask); + size_t simmatRows, simmatCols, simmatRows2, simmatCols2, maskRows, maskCols; + openAndReadMatrixHeader(simmatFile, simmatRows, simmatCols); + openAndReadMatrixHeader(simmatFile2, simmatRows2, simmatCols2); + openAndReadMatrixHeader(maskFile, maskRows, maskCols); + + if ((simmatRows != maskRows) || (simmatCols != maskCols) || (simmatRows2 != maskRows) || (simmatCols2 != maskCols)) + qFatal("Simmat and Mask size mismatch!"); + + // DET stats + size_t genuineCount(0), impostorCount(0), numNaNs(0); + QHash stats; + + // IET stats + size_t genuineSearchCount(0), impostorSearchCount(0); + QHash ietStats; + + // CMC stats + QVector firstGenuineReturns(simmatRows, 0); + + Mat simmatRow, simmatRow2, maskRow; + simmatRow.create(1, simmatCols, CV_32FC1); + simmatRow2.create(1, simmatCols, CV_32FC1); + maskRow.create(1, maskCols, CV_8UC1); + for (size_t i=0; i::max()), highestGenuine(-std::numeric_limits::max()); + for (size_t j=0; j(0, j); + const BEE::SimmatValue sim_val2 = simmatRow2.at(0, j); + const BEE::SimmatValue sim_val = sim_val1*w1 + sim_val2*w2; + const BEE::MaskValue mask_val = maskRow.at(0, j); + if (mask_val == BEE::DontCare) continue; + if (sim_val != sim_val) { numNaNs++; continue; } + + const bool genuine = (mask_val == BEE::Match); + if (genuine) { + hasGenuine = true; + genuineCount++; + stats[sim_val].truePositive++; + if (sim_val > highestImpostor) + impostorsAboveGenuine = 0; + if (sim_val > highestGenuine) + highestGenuine = sim_val; + } else { + hasImpostor = true; + impostorCount++; + stats[sim_val].falsePositive++; + if (sim_val >= highestGenuine) + impostorsAboveGenuine++; + if (sim_val > highestImpostor) + highestImpostor = sim_val; + } + } + + // CMC + firstGenuineReturns[i] = hasGenuine ? impostorsAboveGenuine + 1 : -impostorsAboveGenuine; + + // IET + if (hasGenuine) { // true search + genuineSearchCount++; + ietStats[highestGenuine].truePositive++; + } else if (hasImpostor) { // false search + impostorSearchCount++; + ietStats[highestImpostor].falsePositive++; + } + } + + if (numNaNs > 0) qWarning("Encountered %ld NaN scores!", numNaNs); + if (genuineCount == 0) qFatal("No genuine scores!"); + if (impostorCount == 0) qFatal("No impostor scores!"); + + // Write Metadata table + QStringList lines; + lines.append("Plot,X,Y"); + lines.append("Metadata,"+QString::number(simmatCols)+",Gallery"); + lines.append("Metadata,"+QString::number(simmatRows)+",Probe"); + lines.append("Metadata,"+QString::number(genuineCount)+",Genuine"); + lines.append("Metadata,"+QString::number(impostorCount)+",Impostor"); + lines.append("Metadata,"+QString::number(simmatCols*simmatRows-(genuineCount+impostorCount))+",Ignored"); + + // SD points + size_t sdPoints = qMin(qMin((size_t)Max_Points, genuineCount), impostorCount); + size_t genuineRate = genuineCount / sdPoints; + size_t impostorRate = impostorCount / sdPoints; + size_t genuinesSampled(0), sampleNextGenuine(0), impostorsSampled(0), sampleNextImpostor(0); + + // DET and SD sampling + size_t fps(0), previous_fps(0), tps(0), previous_tps(0); + QList thresholds = stats.keys(); + std::sort(thresholds.begin(), thresholds.end()); + std::reverse(thresholds.begin(), thresholds.end()); + QList operatingPoints; + for (int i=0; i previous_fps) && tps > previous_tps) { + operatingPoints.append(OperatingPoint(thresholds[i], double(fps)/impostorCount, double(tps)/genuineCount)); + previous_fps = fps; + previous_tps = tps; + } + + // Genuine score sampling + if ((genuinesSampled < sdPoints) && (counter.truePositive > 0)) { + for (int sample=0; sample= sdPoints) + break; + } + } + } + + // Impostor score sampling + if ((impostorsSampled < sdPoints) && (counter.falsePositive > 0)) { + for (int sample=0; sample= sdPoints) + break; + } + } + } + } + if (operatingPoints.size() == 0) operatingPoints.append(OperatingPoint(1, 1, 1)); + if (operatingPoints.size() == 1) operatingPoints.prepend(OperatingPoint(0, 0, 0)); + if (operatingPoints.size() > 2) operatingPoints.takeLast(); // Remove point (1,1) + + // IET + QList searchOperatingPoints; + if (genuineSearchCount > 0 && impostorSearchCount > 0) { + fps = 0; tps = 0; + QList thresholds = ietStats.keys(); + std::sort(thresholds.begin(), thresholds.end()); + std::reverse(thresholds.begin(), thresholds.end()); + for (int i=0; i 2) searchOperatingPoints.takeLast(); // Remove point (1,1) + + + // Write Detection Error Tradeoff (DET), PRE, REC, Identification Error Tradeoff (IET) + float expFAR = std::max(ceil(log10(impostorCount)), 1.0); + float expFRR = std::max(ceil(log10(genuineCount)), 1.0); + float expFPIR = std::max(ceil(log10(impostorSearchCount)), 1.0); + + float FARstep = expFAR / (float)(Max_Points - 1); + float FRRstep = expFRR / (float)(Max_Points - 1); + float FPIRstep = expFPIR / (float)(Max_Points - 1); + + for (int i=0; i() << 1e-6 << 1e-5 << 1e-4 << 1e-3 << 1e-2 << 1e-1) + lines.append(qPrintable(QString("FF,%1,%2").arg( + QString::number(FAR, 'f'), + QString::number(1-getOperatingPoint(operatingPoints, "FAR", FAR).TAR, 'f', 6)))); + + // Write FAR@TAR Table (FT) + foreach (float TAR, QList() << 0.4 << 0.5 << 0.65 << 0.75 << 0.85 << 0.95) + lines.append(qPrintable(QString("FT,%1,%2").arg( + QString::number(TAR, 'f', 2), + QString::number(getOperatingPoint(operatingPoints, "TAR", TAR).FAR, 'f', 3)))); + + // Write FAR@Score Table (SF) and TAR@Score table (ST) + foreach(const float score, QList() << 0.05 << 0.1 << 0.15 << 0.2 << 0.25 << 0.3 << 0.35 << 0.4 << 0.45 << 0.5 + << 0.55 << 0.6 << 0.65 << 0.7 << 0.75 << 0.8 << 0.85 << 0.9 << 0.95) { + const OperatingPoint op = getOperatingPoint(operatingPoints, "Score", score); + lines.append(qPrintable(QString("SF,%1,%2").arg( + QString::number(score, 'f', 2), + QString::number(op.FAR)))); + lines.append(qPrintable(QString("ST,%1,%2").arg( + QString::number(score, 'f', 2), + QString::number(op.TAR)))); + } + + // Write FAR/TAR Bar Chart (BC) + float result = 0; + lines.append(qPrintable(QString("BC,0.0001,%1").arg(QString::number(getOperatingPoint(operatingPoints, "FAR", 0.0001).TAR, 'f', 3)))); + lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, "FAR", 0.001).TAR, 'f', 3)))); + + QtUtils::writeFile(csv, lines); + + foreach (float FAR, QList() << 1e-1 << 1e-2 << 1e-3 << 1e-4 << 1e-5 << 1e-6) { + const OperatingPoint op = getOperatingPoint(operatingPoints, "FAR", FAR); + printf("TAR & Similarity @ FAR = %.0e: %.4f %.3f\n", FAR, op.TAR, op.score); + } + printf("\n"); + foreach (float FPIR, QList() << 0.1 << 0.01) { + const OperatingPoint op = getOperatingPoint(searchOperatingPoints, "FAR", FPIR); + printf("FNIR @ FPIR = %.0e: %.3f\n", FPIR, 1-op.TAR); + } + printf("\n"); + foreach (const int Report_Retrieval, QList() << 1 << 5 << 10 << 20 << 50 << 100) + printf("Retrieval Rate @ Rank = %d: %.3f\n", Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval)); + + return result; +} + void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty, QString truthProperty) { qDebug("Evaluating classification of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); diff --git a/openbr/core/eval.h b/openbr/core/eval.h index d41f4a510..2dd539f0e 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -24,10 +24,13 @@ namespace br { float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 + float EvaluateFused(const QString &simmat, const QString &simmat2, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.7, float w2=0.3); // Returns TAR @ FAR = 0.001 + float EvaluateFused2(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.35, float w2=0.35, float w3=0.3); // Returns TAR @ FAR = 0.001 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0); - float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0); + float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0, const QString &target2 = "", const QString &target3 = ""); void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001 float InplaceEval(const QString &simmat, const QString &mask, const QString &csv); + float InplaceEvalFused(const QString &simmat, const QString &simmat22, const QString &mask, const QString &csv, const float w1, const float w2); void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0, const QString &label = "", const float true_positive_threshold = 0.5f); // Return average overlap diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 3b8c1815b..3d26c2b4d 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -106,6 +106,16 @@ float br_eval(const char *simmat, const char *mask, const char *csv, int matches return Evaluate(simmat, mask, csv, matches); } +float br_eval_fused(const char *simmat, const char *simmat2, const char *mask, const char *csv, int matches, float w1, float w2) +{ + return EvaluateFused(simmat, simmat2, mask, csv, matches, w1, w2); +} + +float br_eval_fused2(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv, int matches, float w1, float w2, float w3) +{ + return EvaluateFused2(simmat, simmat2, simmat3, mask, csv, matches, w1, w2, w3); +} + void br_assert_eval(const char *simmat, const char *mask, const float accuracy) { assertEval(simmat, mask, accuracy); diff --git a/openbr/openbr.h b/openbr/openbr.h index f8ecdb1ee..3352d59a0 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -49,6 +49,8 @@ BR_EXPORT void br_enroll_n(int num_inputs, const char *inputs[], const char *gal BR_EXPORT void br_project(const char *input, const char *output); BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = "", int matches = 0); +BR_EXPORT float br_eval_fused(const char *simmat, const char *simmat2, const char *mask, const char *csv = "", int matches = 0, float w1 = 0.7, float w2 = 0.3); +BR_EXPORT float br_eval_fused2(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv = "", int matches = 0, float w1 = 0.35, float w2 = 0.35, float w3 = 0.3); BR_EXPORT void br_assert_eval(const char *simmat, const char *mask, const float accuracy); From 79a4c1935297e4f041e135b7e3ed1b5970ce17a9 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 00:21:26 +0000 Subject: [PATCH 02/12] adding score fusion of two simmats --- app/br/br.cpp | 4 ++-- openbr/core/eval.cpp | 16 ++++++++++++++-- openbr/core/eval.h | 4 ++-- openbr/openbr.cpp | 12 +++++------- openbr/openbr.h | 3 +-- 5 files changed, 24 insertions(+), 15 deletions(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index dceaedee8..6b1c33b33 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -126,9 +126,9 @@ class FakeMain : public QRunnable } else if (!strcmp(fun, "evalfused")) { check((parc ==6 || (parc == 8)), "Incorrect parameter count for 'evalfused'."); if (parc == 6) { - br_eval_fused(parv[0], parv[1], parv[2], parv[3], 0, atof(parv[4]), atof(parv[5])); + br_eval_fused(parv[0], parv[1], "", parv[2], parv[3], 0, atof(parv[4]), atof(parv[5]), 0); } else { - br_eval_fused2(parv[0], parv[1], parv[2], parv[3], parv[4], 0, atof(parv[5]), atof(parv[6]), atof(parv[7])); + br_eval_fused(parv[0], parv[1], parv[2], parv[3], parv[4], 0, atof(parv[5]), atof(parv[6]), atof(parv[7])); } } else if (!strcmp(fun, "plot")) { check(parc >= 2, "Incorrect parameter count for 'plot'."); diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 09e140476..3d67bd2f0 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -188,7 +188,7 @@ float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsi return Evaluate(scores, truth, csv, target, query, matches); } -float EvaluateFused(const QString &simmat, const QString &simmat2, const QString &mask, const File &csv, unsigned int matches, float w1, float w2) +float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask, const File &csv, unsigned int matches, float w1, float w2) { qDebug("Evaluating %s %s%s%s", qPrintable(simmat), @@ -217,6 +217,18 @@ float EvaluateFused(const QString &simmat, const QString &simmat2, const QString scores = scores*w1 + scores2*w2; + // float lowerBound = 0; + // float upperBound = 1; + + // for (int i = 0; i < scores.rows; i++) { + // for (int j = 0; j < scores.cols; j++) { + // if (scores.at(i, j) >= lowerBound && scores.at(i, j) <= upperBound) { + // scores.at(i, j) = w1*scores.at(i, j) + w2*scores2.at(i, j); + // } + // } + // } + + // Read mask matrix Mat truth; if (mask.isEmpty()) { @@ -237,7 +249,7 @@ float EvaluateFused(const QString &simmat, const QString &simmat2, const QString return Evaluate(scores, truth, csv, target, query, matches, target2, ""); } -float EvaluateFused2(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask, const File &csv, unsigned int matches, float w1, float w2, float w3) +float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask, const File &csv, unsigned int matches, float w1, float w2, float w3) { qDebug("Evaluating %s %s%s%s%s", qPrintable(simmat), diff --git a/openbr/core/eval.h b/openbr/core/eval.h index 2dd539f0e..afbc5189a 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -24,8 +24,8 @@ namespace br { float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 - float EvaluateFused(const QString &simmat, const QString &simmat2, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.7, float w2=0.3); // Returns TAR @ FAR = 0.001 - float EvaluateFused2(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.35, float w2=0.35, float w3=0.3); // Returns TAR @ FAR = 0.001 + float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.7, float w2=0.3); // Returns TAR @ FAR = 0.001 + float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.35, float w2=0.35, float w3=0.3); // Returns TAR @ FAR = 0.001 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0); float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0, const QString &target2 = "", const QString &target3 = ""); void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001 diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 3d26c2b4d..a1e352e65 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -106,14 +106,12 @@ float br_eval(const char *simmat, const char *mask, const char *csv, int matches return Evaluate(simmat, mask, csv, matches); } -float br_eval_fused(const char *simmat, const char *simmat2, const char *mask, const char *csv, int matches, float w1, float w2) +float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv, int matches, float w1, float w2, float w3) { - return EvaluateFused(simmat, simmat2, mask, csv, matches, w1, w2); -} - -float br_eval_fused2(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv, int matches, float w1, float w2, float w3) -{ - return EvaluateFused2(simmat, simmat2, simmat3, mask, csv, matches, w1, w2, w3); + if (NULL == *simmat3) { + return Evaluate(simmat, simmat2, mask, csv, matches, w1, w2); + } + return Evaluate(simmat, simmat2, simmat3, mask, csv, matches, w1, w2, w3); } void br_assert_eval(const char *simmat, const char *mask, const float accuracy) diff --git a/openbr/openbr.h b/openbr/openbr.h index 3352d59a0..805f43c9d 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -49,8 +49,7 @@ BR_EXPORT void br_enroll_n(int num_inputs, const char *inputs[], const char *gal BR_EXPORT void br_project(const char *input, const char *output); BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = "", int matches = 0); -BR_EXPORT float br_eval_fused(const char *simmat, const char *simmat2, const char *mask, const char *csv = "", int matches = 0, float w1 = 0.7, float w2 = 0.3); -BR_EXPORT float br_eval_fused2(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv = "", int matches = 0, float w1 = 0.35, float w2 = 0.35, float w3 = 0.3); +BR_EXPORT float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv = "", int matches = 0, float w1 = 0.7, float w2 = 0.3, float w3 = 0.3); BR_EXPORT void br_assert_eval(const char *simmat, const char *mask, const float accuracy); From db1bfc2ee9b33ab51b2db5a110ff1b2dd7e1ae3d Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 20:23:22 +0000 Subject: [PATCH 03/12] adding score fusion of 2 to 3 simmats --- app/br/br.cpp | 8 +- openbr/core/eval.cpp | 317 +++++++------------------------------------ openbr/core/eval.h | 7 +- openbr/openbr.cpp | 8 +- openbr/openbr.h | 2 +- 5 files changed, 62 insertions(+), 280 deletions(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index 6b1c33b33..2585c7640 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -124,11 +124,11 @@ class FakeMain : public QRunnable br_eval(parv[0], parv[1], parv[2], atoi(parv[3])); } } else if (!strcmp(fun, "evalfused")) { - check((parc ==6 || (parc == 8)), "Incorrect parameter count for 'evalfused'."); - if (parc == 6) { - br_eval_fused(parv[0], parv[1], "", parv[2], parv[3], 0, atof(parv[4]), atof(parv[5]), 0); + check((parc == 8 || (parc == 10)), "Incorrect parameter count for 'evalfused'."); + if (parc == 8) { + br_eval_fused(parv[0], parv[1], "", parv[2], parv[3], 0, atof(parv[4]), atof(parv[5]), 0, atof(parv[6]), atof(parv[7])); } else { - br_eval_fused(parv[0], parv[1], parv[2], parv[3], parv[4], 0, atof(parv[5]), atof(parv[6]), atof(parv[7])); + br_eval_fused(parv[0], parv[1], parv[2], parv[3], parv[4], 0, atof(parv[5]), atof(parv[6]), atof(parv[7]), atof(parv[8]), atof(parv[9])); } } else if (!strcmp(fun, "plot")) { check(parc >= 2, "Incorrect parameter count for 'plot'."); diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 3d67bd2f0..4ed458c7f 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -188,7 +188,7 @@ float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsi return Evaluate(scores, truth, csv, target, query, matches); } -float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask, const File &csv, unsigned int matches, float w1, float w2) +float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask, const File &csv, unsigned int matches, float w1, float w2, float lowerBound, float upperBound) { qDebug("Evaluating %s %s%s%s", qPrintable(simmat), @@ -215,20 +215,6 @@ float Evaluate(const QString &simmat, const QString &simmat2, const QString &mas scores2 = format->read(); } - scores = scores*w1 + scores2*w2; - - // float lowerBound = 0; - // float upperBound = 1; - - // for (int i = 0; i < scores.rows; i++) { - // for (int j = 0; j < scores.cols; j++) { - // if (scores.at(i, j) >= lowerBound && scores.at(i, j) <= upperBound) { - // scores.at(i, j) = w1*scores.at(i, j) + w2*scores2.at(i, j); - // } - // } - // } - - // Read mask matrix Mat truth; if (mask.isEmpty()) { @@ -246,10 +232,33 @@ float Evaluate(const QString &simmat, const QString &simmat2, const QString &mas truth = format->read(); } + float count_total = 0; + float count_fused = 0; + for (int i = 0; i < scores.rows; i++) { + for (int j = 0; j < scores.cols; j++) { + const BEE::MaskValue mask_val = truth.at(i,j); + const BEE::SimmatValue simmat_val = scores.at(i,j); + const BEE::SimmatValue simmat_val2 = scores2.at(i,j); + + if (mask_val == BEE::DontCare) continue; + if (simmat_val != simmat_val) continue; + + if (simmat_val >= lowerBound && simmat_val <= upperBound) { + scores.at(i,j) = w1*simmat_val + w2*simmat_val2; + count_fused += 1; + } + count_total += 1; + } + } + qDebug("fused %f percent of comparisons w/ scores falling between [%f, %f]", + count_fused/count_total*100, + lowerBound, + upperBound); + return Evaluate(scores, truth, csv, target, query, matches, target2, ""); } -float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask, const File &csv, unsigned int matches, float w1, float w2, float w3) +float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask, const File &csv, unsigned int matches, float w1, float w2, float w3, float lowerBound, float upperBound) { qDebug("Evaluating %s %s%s%s%s", qPrintable(simmat), @@ -286,8 +295,6 @@ float Evaluate(const QString &simmat, const QString &simmat2, const QString &sim scores3 = format->read(); } - scores = scores*w1 + scores2*w2 + scores3*w3; - // Read mask matrix Mat truth; if (mask.isEmpty()) { @@ -305,6 +312,30 @@ float Evaluate(const QString &simmat, const QString &simmat2, const QString &sim truth = format->read(); } + float count_total = 0; + float count_fused = 0; + for (int i = 0; i < scores.rows; i++) { + for (int j = 0; j < scores.cols; j++) { + const BEE::MaskValue mask_val = truth.at(i,j); + const BEE::SimmatValue simmat_val = scores.at(i,j); + const BEE::SimmatValue simmat_val2 = scores2.at(i,j); + const BEE::SimmatValue simmat_val3 = scores3.at(i,j); + + if (mask_val == BEE::DontCare) continue; + if (simmat_val != simmat_val) continue; + + if (simmat_val >= lowerBound && simmat_val <= upperBound) { + scores.at(i,j) = w1*simmat_val + w2*simmat_val2 + w3*simmat_val3; + count_fused += 1; + } + count_total += 1; + } + } + qDebug("fused %f percent of comparisons w/ scores falling between [%f, %f]", + count_fused/count_total*100, + lowerBound, + upperBound); + return Evaluate(scores, truth, csv, target, query, matches, target2, target3); } @@ -879,256 +910,6 @@ float InplaceEval(const QString &simmat, const QString &mask, const QString &csv return result; } -float InplaceEvalFused(const QString &simmat, const QString &simmat2, const QString &mask, const QString &csv, const float w1, const float w2) -{ - QFile simmatFile(simmat), simmatFile2(simmat2), maskFile(mask); - size_t simmatRows, simmatCols, simmatRows2, simmatCols2, maskRows, maskCols; - openAndReadMatrixHeader(simmatFile, simmatRows, simmatCols); - openAndReadMatrixHeader(simmatFile2, simmatRows2, simmatCols2); - openAndReadMatrixHeader(maskFile, maskRows, maskCols); - - if ((simmatRows != maskRows) || (simmatCols != maskCols) || (simmatRows2 != maskRows) || (simmatCols2 != maskCols)) - qFatal("Simmat and Mask size mismatch!"); - - // DET stats - size_t genuineCount(0), impostorCount(0), numNaNs(0); - QHash stats; - - // IET stats - size_t genuineSearchCount(0), impostorSearchCount(0); - QHash ietStats; - - // CMC stats - QVector firstGenuineReturns(simmatRows, 0); - - Mat simmatRow, simmatRow2, maskRow; - simmatRow.create(1, simmatCols, CV_32FC1); - simmatRow2.create(1, simmatCols, CV_32FC1); - maskRow.create(1, maskCols, CV_8UC1); - for (size_t i=0; i::max()), highestGenuine(-std::numeric_limits::max()); - for (size_t j=0; j(0, j); - const BEE::SimmatValue sim_val2 = simmatRow2.at(0, j); - const BEE::SimmatValue sim_val = sim_val1*w1 + sim_val2*w2; - const BEE::MaskValue mask_val = maskRow.at(0, j); - if (mask_val == BEE::DontCare) continue; - if (sim_val != sim_val) { numNaNs++; continue; } - - const bool genuine = (mask_val == BEE::Match); - if (genuine) { - hasGenuine = true; - genuineCount++; - stats[sim_val].truePositive++; - if (sim_val > highestImpostor) - impostorsAboveGenuine = 0; - if (sim_val > highestGenuine) - highestGenuine = sim_val; - } else { - hasImpostor = true; - impostorCount++; - stats[sim_val].falsePositive++; - if (sim_val >= highestGenuine) - impostorsAboveGenuine++; - if (sim_val > highestImpostor) - highestImpostor = sim_val; - } - } - - // CMC - firstGenuineReturns[i] = hasGenuine ? impostorsAboveGenuine + 1 : -impostorsAboveGenuine; - - // IET - if (hasGenuine) { // true search - genuineSearchCount++; - ietStats[highestGenuine].truePositive++; - } else if (hasImpostor) { // false search - impostorSearchCount++; - ietStats[highestImpostor].falsePositive++; - } - } - - if (numNaNs > 0) qWarning("Encountered %ld NaN scores!", numNaNs); - if (genuineCount == 0) qFatal("No genuine scores!"); - if (impostorCount == 0) qFatal("No impostor scores!"); - - // Write Metadata table - QStringList lines; - lines.append("Plot,X,Y"); - lines.append("Metadata,"+QString::number(simmatCols)+",Gallery"); - lines.append("Metadata,"+QString::number(simmatRows)+",Probe"); - lines.append("Metadata,"+QString::number(genuineCount)+",Genuine"); - lines.append("Metadata,"+QString::number(impostorCount)+",Impostor"); - lines.append("Metadata,"+QString::number(simmatCols*simmatRows-(genuineCount+impostorCount))+",Ignored"); - - // SD points - size_t sdPoints = qMin(qMin((size_t)Max_Points, genuineCount), impostorCount); - size_t genuineRate = genuineCount / sdPoints; - size_t impostorRate = impostorCount / sdPoints; - size_t genuinesSampled(0), sampleNextGenuine(0), impostorsSampled(0), sampleNextImpostor(0); - - // DET and SD sampling - size_t fps(0), previous_fps(0), tps(0), previous_tps(0); - QList thresholds = stats.keys(); - std::sort(thresholds.begin(), thresholds.end()); - std::reverse(thresholds.begin(), thresholds.end()); - QList operatingPoints; - for (int i=0; i previous_fps) && tps > previous_tps) { - operatingPoints.append(OperatingPoint(thresholds[i], double(fps)/impostorCount, double(tps)/genuineCount)); - previous_fps = fps; - previous_tps = tps; - } - - // Genuine score sampling - if ((genuinesSampled < sdPoints) && (counter.truePositive > 0)) { - for (int sample=0; sample= sdPoints) - break; - } - } - } - - // Impostor score sampling - if ((impostorsSampled < sdPoints) && (counter.falsePositive > 0)) { - for (int sample=0; sample= sdPoints) - break; - } - } - } - } - if (operatingPoints.size() == 0) operatingPoints.append(OperatingPoint(1, 1, 1)); - if (operatingPoints.size() == 1) operatingPoints.prepend(OperatingPoint(0, 0, 0)); - if (operatingPoints.size() > 2) operatingPoints.takeLast(); // Remove point (1,1) - - // IET - QList searchOperatingPoints; - if (genuineSearchCount > 0 && impostorSearchCount > 0) { - fps = 0; tps = 0; - QList thresholds = ietStats.keys(); - std::sort(thresholds.begin(), thresholds.end()); - std::reverse(thresholds.begin(), thresholds.end()); - for (int i=0; i 2) searchOperatingPoints.takeLast(); // Remove point (1,1) - - - // Write Detection Error Tradeoff (DET), PRE, REC, Identification Error Tradeoff (IET) - float expFAR = std::max(ceil(log10(impostorCount)), 1.0); - float expFRR = std::max(ceil(log10(genuineCount)), 1.0); - float expFPIR = std::max(ceil(log10(impostorSearchCount)), 1.0); - - float FARstep = expFAR / (float)(Max_Points - 1); - float FRRstep = expFRR / (float)(Max_Points - 1); - float FPIRstep = expFPIR / (float)(Max_Points - 1); - - for (int i=0; i() << 1e-6 << 1e-5 << 1e-4 << 1e-3 << 1e-2 << 1e-1) - lines.append(qPrintable(QString("FF,%1,%2").arg( - QString::number(FAR, 'f'), - QString::number(1-getOperatingPoint(operatingPoints, "FAR", FAR).TAR, 'f', 6)))); - - // Write FAR@TAR Table (FT) - foreach (float TAR, QList() << 0.4 << 0.5 << 0.65 << 0.75 << 0.85 << 0.95) - lines.append(qPrintable(QString("FT,%1,%2").arg( - QString::number(TAR, 'f', 2), - QString::number(getOperatingPoint(operatingPoints, "TAR", TAR).FAR, 'f', 3)))); - - // Write FAR@Score Table (SF) and TAR@Score table (ST) - foreach(const float score, QList() << 0.05 << 0.1 << 0.15 << 0.2 << 0.25 << 0.3 << 0.35 << 0.4 << 0.45 << 0.5 - << 0.55 << 0.6 << 0.65 << 0.7 << 0.75 << 0.8 << 0.85 << 0.9 << 0.95) { - const OperatingPoint op = getOperatingPoint(operatingPoints, "Score", score); - lines.append(qPrintable(QString("SF,%1,%2").arg( - QString::number(score, 'f', 2), - QString::number(op.FAR)))); - lines.append(qPrintable(QString("ST,%1,%2").arg( - QString::number(score, 'f', 2), - QString::number(op.TAR)))); - } - - // Write FAR/TAR Bar Chart (BC) - float result = 0; - lines.append(qPrintable(QString("BC,0.0001,%1").arg(QString::number(getOperatingPoint(operatingPoints, "FAR", 0.0001).TAR, 'f', 3)))); - lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, "FAR", 0.001).TAR, 'f', 3)))); - - QtUtils::writeFile(csv, lines); - - foreach (float FAR, QList() << 1e-1 << 1e-2 << 1e-3 << 1e-4 << 1e-5 << 1e-6) { - const OperatingPoint op = getOperatingPoint(operatingPoints, "FAR", FAR); - printf("TAR & Similarity @ FAR = %.0e: %.4f %.3f\n", FAR, op.TAR, op.score); - } - printf("\n"); - foreach (float FPIR, QList() << 0.1 << 0.01) { - const OperatingPoint op = getOperatingPoint(searchOperatingPoints, "FAR", FPIR); - printf("FNIR @ FPIR = %.0e: %.3f\n", FPIR, 1-op.TAR); - } - printf("\n"); - foreach (const int Report_Retrieval, QList() << 1 << 5 << 10 << 20 << 50 << 100) - printf("Retrieval Rate @ Rank = %d: %.3f\n", Report_Retrieval, getCMC(firstGenuineReturns, Report_Retrieval)); - - return result; -} - void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty, QString truthProperty) { qDebug("Evaluating classification of %s against %s", qPrintable(predictedGallery), qPrintable(truthGallery)); diff --git a/openbr/core/eval.h b/openbr/core/eval.h index afbc5189a..375ebb0ff 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -24,14 +24,13 @@ namespace br { float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 - float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.7, float w2=0.3); // Returns TAR @ FAR = 0.001 - float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.35, float w2=0.35, float w3=0.3); // Returns TAR @ FAR = 0.001 + float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.7, float w2=0.3, float lowerBound=-1e6, float upperBound=1e6); // Returns TAR @ FAR = 0.001 + float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.35, float w2=0.35, float w3=0.3, float lowerBound=-1e6, float upperBound=1e6); // Returns TAR @ FAR = 0.001 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0); float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0, const QString &target2 = "", const QString &target3 = ""); void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001 float InplaceEval(const QString &simmat, const QString &mask, const QString &csv); - float InplaceEvalFused(const QString &simmat, const QString &simmat22, const QString &mask, const QString &csv, const float w1, const float w2); - + void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0, const QString &label = "", const float true_positive_threshold = 0.5f); // Return average overlap float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index a1e352e65..68c8aceba 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -106,12 +106,14 @@ float br_eval(const char *simmat, const char *mask, const char *csv, int matches return Evaluate(simmat, mask, csv, matches); } -float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv, int matches, float w1, float w2, float w3) +float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv, int matches, float w1, float w2, float w3, float lowerBound, float upperBound) { if (NULL == *simmat3) { - return Evaluate(simmat, simmat2, mask, csv, matches, w1, w2); + printf("fusing 2 simmats..."); + return Evaluate(simmat, simmat2, mask, csv, matches, w1, w2, lowerBound, upperBound); } - return Evaluate(simmat, simmat2, simmat3, mask, csv, matches, w1, w2, w3); + printf("fusing 3 simmats..."); + return Evaluate(simmat, simmat2, simmat3, mask, csv, matches, w1, w2, w3, lowerBound, upperBound); } void br_assert_eval(const char *simmat, const char *mask, const float accuracy) diff --git a/openbr/openbr.h b/openbr/openbr.h index 805f43c9d..90c9bc39b 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -49,7 +49,7 @@ BR_EXPORT void br_enroll_n(int num_inputs, const char *inputs[], const char *gal BR_EXPORT void br_project(const char *input, const char *output); BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = "", int matches = 0); -BR_EXPORT float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv = "", int matches = 0, float w1 = 0.7, float w2 = 0.3, float w3 = 0.3); +BR_EXPORT float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv = "", int matches = 0, float w1 = 0.7, float w2 = 0.3, float w3 = 0.3, float lowerBound = -1e6, float upperBound = 1e6); BR_EXPORT void br_assert_eval(const char *simmat, const char *mask, const float accuracy); From 6f07f0c7527230703f37c875ffce4f9732938ee5 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 20:34:22 +0000 Subject: [PATCH 04/12] updating evalfused arguments description --- app/br/br.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index 2585c7640..20115bce4 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -286,7 +286,7 @@ class FakeMain : public QRunnable "-enroll ... {output_gallery}\n" "-compare [{output}]\n" "-eval [] [{csv}] [{matches}]\n" - "-evalfused [] [{csv}] [{matches}]\n" + "-evalfused [] [] [{csv}] [] \n" "-plot ... {destination}\n" "\n" "==== Other Commands ====\n" From 700ee000e70d9b4032cc6bd46b6dc72ae0cdb2f0 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 20:37:56 +0000 Subject: [PATCH 05/12] removing print statements --- openbr/openbr.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 68c8aceba..3174d8dcc 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -109,10 +109,8 @@ float br_eval(const char *simmat, const char *mask, const char *csv, int matches float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv, int matches, float w1, float w2, float w3, float lowerBound, float upperBound) { if (NULL == *simmat3) { - printf("fusing 2 simmats..."); return Evaluate(simmat, simmat2, mask, csv, matches, w1, w2, lowerBound, upperBound); } - printf("fusing 3 simmats..."); return Evaluate(simmat, simmat2, simmat3, mask, csv, matches, w1, w2, w3, lowerBound, upperBound); } From f684c2757cb16249a01397ab6253555e999ec48b Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 20:51:06 +0000 Subject: [PATCH 06/12] propagating change from Commit 54c0e6e --- openbr/core/eval.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 4ed458c7f..5f0b0332e 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -452,8 +452,7 @@ float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QStrin index++; } - if ((falsePositives > previousFalsePositives) && - (truePositives > previousTruePositives)) { + if (truePositives > previousTruePositives) { operatingPoints.append(OperatingPoint(thresh, float(falsePositives)/impostorCount, float(truePositives)/genuineCount)); if (EERIndex == 0) { From 234528bf511c499f847209217a667ef5d957ba6b Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 20:51:57 +0000 Subject: [PATCH 07/12] fixed spacing --- app/br/br.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index 20115bce4..7486d5ad6 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -135,7 +135,6 @@ class FakeMain : public QRunnable br_plot(parc-1, parv, parv[parc-1], true); } - // Secondary Tasks else if (!strcmp(fun, "fuse")) { check(parc >= 4, "Insufficient parameter count for 'fuse'."); From 89f812d7889395fd6c0dbcd42fa98e52f449f2fb Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 23:42:48 +0000 Subject: [PATCH 08/12] converting to QStringList --- app/br/br.cpp | 30 +++++++- openbr/core/eval.cpp | 178 ++++++++++++------------------------------- openbr/core/eval.h | 58 +++++++++++++- openbr/openbr.cpp | 7 +- openbr/openbr.h | 6 +- 5 files changed, 136 insertions(+), 143 deletions(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index 7486d5ad6..c02986ea1 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -50,6 +50,24 @@ * \endcode */ + QVector convertStringToFloatVector(const QString& str) { + QVector result; + QStringList values = str.split(","); + + for (const QString& value : values) { + bool ok; + float num = value.toFloat(&ok); + if (ok) { + result.append(num); + } else { + // Handle conversion error, e.g., by throwing an exception or returning an empty vector + qDebug() << "Error converting string to float:" << value; + } + } + + return result; +} + class FakeMain : public QRunnable { int argc; @@ -124,11 +142,15 @@ class FakeMain : public QRunnable br_eval(parv[0], parv[1], parv[2], atoi(parv[3])); } } else if (!strcmp(fun, "evalfused")) { - check((parc == 8 || (parc == 10)), "Incorrect parameter count for 'evalfused'."); - if (parc == 8) { - br_eval_fused(parv[0], parv[1], "", parv[2], parv[3], 0, atof(parv[4]), atof(parv[5]), 0, atof(parv[6]), atof(parv[7])); + check((parc == 4 || (parc == 6)), "Incorrect parameter count for 'evalfused'."); + if (parc == 4) { + const QStringList simmatList = QString(parv[0]).split(","); + const QVector weights = convertStringToFloatVector(parv[3]); + br_eval_fused(simmatList, parv[1], parv[2], 0, weights, 1e-6, 1e6); } else { - br_eval_fused(parv[0], parv[1], parv[2], parv[3], parv[4], 0, atof(parv[5]), atof(parv[6]), atof(parv[7]), atof(parv[8]), atof(parv[9])); + const QStringList simmatList = QString(parv[0]).split(","); + const QVector weights = convertStringToFloatVector(parv[3]); + br_eval_fused(simmatList, parv[1], parv[2], 0, weights, atof(parv[4]), atof(parv[5])); } } else if (!strcmp(fun, "plot")) { check(parc >= 2, "Incorrect parameter count for 'plot'."); diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index 5f0b0332e..b354e0936 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -148,7 +148,7 @@ static cv::Mat constructMatchingMask(const cv::Mat &scores, const FileList &targ float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv, int partition) { - return Evaluate(scores, constructMatchingMask(scores, target, query, partition), csv, QString(), QString(), 0); + return Evaluate(scores, constructMatchingMask(scores, target, query, partition), csv, QStringList(), QString(), 0); } float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsigned int matches) @@ -185,147 +185,76 @@ float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsi truth = format->read(); } - return Evaluate(scores, truth, csv, target, query, matches); + return Evaluate(scores, truth, csv, {target}, query, matches); } -float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask, const File &csv, unsigned int matches, float w1, float w2, float lowerBound, float upperBound) +float Evaluate(const QStringList &simmats, const QString &mask, const File &csv, unsigned int matches, const QVector &weights, float lowerBound, float upperBound) { - qDebug("Evaluating %s %s%s%s", - qPrintable(simmat), - qPrintable(simmat2), - mask.isEmpty() ? "" : qPrintable(" with " + mask), - csv.name.isEmpty() ? "" : qPrintable(" to " + csv)); - - // Read similarity matrix - QString target, query; - Mat scores; - if (simmat.endsWith(".mtx")) { - scores = BEE::readMatrix(simmat, &target, &query); - } else { - QScopedPointer format(Factory::make(simmat)); - scores = format->read(); - } - - QString target2, query2; - Mat scores2; - if (simmat2.endsWith(".mtx")) { - scores2 = BEE::readMatrix(simmat2, &target2, &query2); - } else { - QScopedPointer format(Factory::make(simmat2)); - scores2 = format->read(); - } - - // Read mask matrix - Mat truth; - if (mask.isEmpty()) { - // Use the galleries specified in the similarity matrix - if (target.isEmpty()) qFatal("Unspecified target gallery."); - if (query.isEmpty()) qFatal("Unspecified query gallery."); - - truth = constructMatchingMask(scores, TemplateList::fromGallery(target).files(), - TemplateList::fromGallery(query).files()); - } else { - File maskFile(mask); - maskFile.set("rows", scores.rows); - maskFile.set("columns", scores.cols); - QScopedPointer format(Factory::make(maskFile)); - truth = format->read(); + if (simmats.size() <= 0) { + throw std::invalid_argument("Must supply at least one simmat."); } - float count_total = 0; - float count_fused = 0; - for (int i = 0; i < scores.rows; i++) { - for (int j = 0; j < scores.cols; j++) { - const BEE::MaskValue mask_val = truth.at(i,j); - const BEE::SimmatValue simmat_val = scores.at(i,j); - const BEE::SimmatValue simmat_val2 = scores2.at(i,j); - - if (mask_val == BEE::DontCare) continue; - if (simmat_val != simmat_val) continue; - - if (simmat_val >= lowerBound && simmat_val <= upperBound) { - scores.at(i,j) = w1*simmat_val + w2*simmat_val2; - count_fused += 1; - } - count_total += 1; - } + if (!weights.isEmpty() && weights.size() != simmats.size()) { + throw std::invalid_argument("The size of weights must match the number of simmats."); } - qDebug("fused %f percent of comparisons w/ scores falling between [%f, %f]", - count_fused/count_total*100, - lowerBound, - upperBound); - return Evaluate(scores, truth, csv, target, query, matches, target2, ""); -} - -float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask, const File &csv, unsigned int matches, float w1, float w2, float w3, float lowerBound, float upperBound) -{ - qDebug("Evaluating %s %s%s%s%s", - qPrintable(simmat), - qPrintable(simmat2), - qPrintable(simmat3), + int n_simmats = simmats.size(); + qDebug("Evaluating %d simmats%s%s", + n_simmats, mask.isEmpty() ? "" : qPrintable(" with " + mask), csv.name.isEmpty() ? "" : qPrintable(" to " + csv)); - // Read similarity matrix - QString target, query; - Mat scores; - if (simmat.endsWith(".mtx")) { - scores = BEE::readMatrix(simmat, &target, &query); - } else { - QScopedPointer format(Factory::make(simmat)); - scores = format->read(); - } - - QString target2, query2; - Mat scores2; - if (simmat2.endsWith(".mtx")) { - scores2 = BEE::readMatrix(simmat2, &target2, &query2); - } else { - QScopedPointer format(Factory::make(simmat2)); - scores2 = format->read(); + QVector scores(n_simmats); + QStringList targets; + for (int i = 0; i < n_simmats; ++i) { + targets.append(""); } + QString query; - QString target3, query3; - Mat scores3; - if (simmat3.endsWith(".mtx")) { - scores3 = BEE::readMatrix(simmat3, &target3, &query3); - } else { - QScopedPointer format(Factory::make(simmat3)); - scores3 = format->read(); + for (int i = 0; i < n_simmats; i++) { + if (simmats[i].endsWith(".mtx")) { + Mat score_mat = BEE::readMatrix(simmats[i], &targets[i], &query); + scores[i] = score_mat; + } else { + QScopedPointer format(Factory::make(simmats[i])); + Mat score_mat = format->read(); + scores[i] = score_mat; + } } // Read mask matrix Mat truth; if (mask.isEmpty()) { // Use the galleries specified in the similarity matrix - if (target.isEmpty()) qFatal("Unspecified target gallery."); + if (targets[0].isEmpty()) qFatal("Unspecified target gallery."); if (query.isEmpty()) qFatal("Unspecified query gallery."); - truth = constructMatchingMask(scores, TemplateList::fromGallery(target).files(), + truth = constructMatchingMask(scores[0], TemplateList::fromGallery(targets[0]).files(), TemplateList::fromGallery(query).files()); } else { File maskFile(mask); - maskFile.set("rows", scores.rows); - maskFile.set("columns", scores.cols); + maskFile.set("rows", scores[0].rows); + maskFile.set("columns", scores[0].cols); QScopedPointer format(Factory::make(maskFile)); truth = format->read(); } float count_total = 0; float count_fused = 0; - for (int i = 0; i < scores.rows; i++) { - for (int j = 0; j < scores.cols; j++) { + for (int i = 0; i < scores[0].rows; i++) { + for (int j = 0; j < scores[0].cols; j++) { const BEE::MaskValue mask_val = truth.at(i,j); - const BEE::SimmatValue simmat_val = scores.at(i,j); - const BEE::SimmatValue simmat_val2 = scores2.at(i,j); - const BEE::SimmatValue simmat_val3 = scores3.at(i,j); - if (mask_val == BEE::DontCare) continue; + + BEE::SimmatValue simmat_val = scores[0].at(i,j); if (simmat_val != simmat_val) continue; if (simmat_val >= lowerBound && simmat_val <= upperBound) { - scores.at(i,j) = w1*simmat_val + w2*simmat_val2 + w3*simmat_val3; + simmat_val *= weights[0]; + for (int k = 1; k < n_simmats; k++) { + simmat_val += weights[k]*scores[k].at(i,j); + } + scores[0].at(i,j) = simmat_val; count_fused += 1; } count_total += 1; @@ -336,12 +265,12 @@ float Evaluate(const QString &simmat, const QString &simmat2, const QString &sim lowerBound, upperBound); - return Evaluate(scores, truth, csv, target, query, matches, target2, target3); + return Evaluate(scores[0], truth, csv, targets, query, matches); } -float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QString &target, const QString &query, unsigned int matches, const QString &target2, const QString &target3) +float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QStringList &targets, const QString &query, unsigned int matches) { - if (target.isEmpty() || query.isEmpty()) matches = 0; + if (targets[0].isEmpty() || query.isEmpty()) matches = 0; if (simmat.size() != mask.size()) qFatal("Similarity matrix (%ix%i) differs in size from mask matrix (%ix%i).", simmat.rows, simmat.cols, mask.rows, mask.cols); @@ -489,7 +418,7 @@ float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QStrin QString filePath = Globals->path; if (matches != 0 && EERIndex != 0) { - const FileList targetFiles = TemplateList::fromGallery(target).files(); + const FileList targetFiles = TemplateList::fromGallery(targets[0]).files(); const FileList queryFiles = TemplateList::fromGallery(query).files(); unsigned int count = 0; for (int i = EERIndex-1; i >= 0; i--) { @@ -574,24 +503,15 @@ float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QStrin // Attempt to read template size from enrolled gallery and write to output CSV size_t maxSize(0); - size_t maxSize2(0); - size_t maxSize3(0); - - if (target.endsWith(".gal") && QFileInfo(target).exists() && target2.endsWith(".gal") && QFileInfo(target2).exists() && target3.endsWith(".gal") && QFileInfo(target3).exists()) { - foreach (const Template &t, TemplateList::fromGallery(target)) maxSize = max(maxSize, t.bytes()); - foreach (const Template &t, TemplateList::fromGallery(target2)) maxSize2 = max(maxSize2, t.bytes()); - foreach (const Template &t, TemplateList::fromGallery(target3)) maxSize3 = max(maxSize3, t.bytes()); - maxSize = maxSize+maxSize2+maxSize3; - lines.append(QString("TS,,%1").arg(QString::number(maxSize))); - } else if (target.endsWith(".gal") && QFileInfo(target).exists() && target2.endsWith(".gal") && QFileInfo(target2).exists()) { - foreach (const Template &t, TemplateList::fromGallery(target)) maxSize = max(maxSize, t.bytes()); - foreach (const Template &t, TemplateList::fromGallery(target2)) maxSize2 = max(maxSize2, t.bytes()); - maxSize = maxSize+maxSize2; - lines.append(QString("TS,,%1").arg(QString::number(maxSize))); - } else if (target.endsWith(".gal") && QFileInfo(target).exists()) { - foreach (const Template &t, TemplateList::fromGallery(target)) maxSize = max(maxSize, t.bytes()); - lines.append(QString("TS,,%1").arg(QString::number(maxSize))); + // size_t maxSize2(0); + // size_t maxSize3(0); + + for (int i=0; i +// #include +// #include "openbr/openbr_plugin.h" + +// namespace br +// { +// float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 +// float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.7, float w2=0.3, float lowerBound=-1e6, float upperBound=1e6); // Returns TAR @ FAR = 0.001 +// float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.35, float w2=0.35, float w3=0.3, float lowerBound=-1e6, float upperBound=1e6); // Returns TAR @ FAR = 0.001 +// float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0); +// float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0, const QString &target2 = "", const QString &target3 = ""); +// void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001 +// float InplaceEval(const QString &simmat, const QString &mask, const QString &csv); + +// void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); +// float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0, const QString &label = "", const float true_positive_threshold = 0.5f); // Return average overlap +// float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error +// void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "", bool generatePlots = true); +// void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = ""); +// void EvalEER(const QString &predictedXML, const QString gt_property = "", const QString distribution_property = "", const QString &csv = ""); +// struct Candidate +// { +// size_t index; +// float similarity; + +// Candidate() {} +// Candidate(size_t index_, float similarity_) +// : index(index_), similarity(similarity_) {} +// }; +// } + +// #endif // BR_EVAL_H + /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright 2012 The MITRE Corporation * * * @@ -24,10 +76,10 @@ namespace br { float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 - float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.7, float w2=0.3, float lowerBound=-1e6, float upperBound=1e6); // Returns TAR @ FAR = 0.001 - float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.35, float w2=0.35, float w3=0.3, float lowerBound=-1e6, float upperBound=1e6); // Returns TAR @ FAR = 0.001 + float Evaluate(const QStringList &simmats, const QString &mask = "", const File &csv = "", unsigned int matches = 0, const QVector &weights = QVector(), float lowerBound = -1e6, float upperBound = 1e6); // Returns TAR @ FAR = 0.001 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0); - float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0, const QString &target2 = "", const QString &target3 = ""); + float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QStringList &targets = QStringList(), const QString &query = "", unsigned int matches = 0); + void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001 float InplaceEval(const QString &simmat, const QString &mask, const QString &csv); diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 3174d8dcc..4547a65c4 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -106,12 +106,9 @@ float br_eval(const char *simmat, const char *mask, const char *csv, int matches return Evaluate(simmat, mask, csv, matches); } -float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv, int matches, float w1, float w2, float w3, float lowerBound, float upperBound) +float br_eval_fused(const QStringList &simmats, const char *mask, const char *csv, int matches, const QVector &weights, float lowerBound, float upperBound) { - if (NULL == *simmat3) { - return Evaluate(simmat, simmat2, mask, csv, matches, w1, w2, lowerBound, upperBound); - } - return Evaluate(simmat, simmat2, simmat3, mask, csv, matches, w1, w2, w3, lowerBound, upperBound); + return Evaluate(simmats, mask, csv, matches, weights, lowerBound, upperBound); } void br_assert_eval(const char *simmat, const char *mask, const float accuracy) diff --git a/openbr/openbr.h b/openbr/openbr.h index 90c9bc39b..4cbd9220c 100644 --- a/openbr/openbr.h +++ b/openbr/openbr.h @@ -18,12 +18,13 @@ #define OPENBR_H #include +#include +#include #ifdef __cplusplus extern "C" { #endif - BR_EXPORT const char *br_about(); BR_EXPORT void br_cat(int num_input_galleries, const char *input_galleries[], const char *output_gallery); @@ -49,7 +50,8 @@ BR_EXPORT void br_enroll_n(int num_inputs, const char *inputs[], const char *gal BR_EXPORT void br_project(const char *input, const char *output); BR_EXPORT float br_eval(const char *simmat, const char *mask, const char *csv = "", int matches = 0); -BR_EXPORT float br_eval_fused(const char *simmat, const char *simmat2, const char *simmat3, const char *mask, const char *csv = "", int matches = 0, float w1 = 0.7, float w2 = 0.3, float w3 = 0.3, float lowerBound = -1e6, float upperBound = 1e6); + +BR_EXPORT float br_eval_fused(const QStringList &simmats, const char *mask, const char *csv = "", int matches = 0, const QVector &weights = {}, float lowerBound = -1e6, float upperBound = 1e6); BR_EXPORT void br_assert_eval(const char *simmat, const char *mask, const float accuracy); From b15ff559a9109bda7b2a07c2abf8ee9635641313 Mon Sep 17 00:00:00 2001 From: Steven Date: Thu, 5 Dec 2024 23:54:00 +0000 Subject: [PATCH 09/12] fixing default lowerBound and maxSize for fusion --- app/br/br.cpp | 2 +- openbr/core/eval.cpp | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/br/br.cpp b/app/br/br.cpp index c02986ea1..67eee591f 100644 --- a/app/br/br.cpp +++ b/app/br/br.cpp @@ -146,7 +146,7 @@ class FakeMain : public QRunnable if (parc == 4) { const QStringList simmatList = QString(parv[0]).split(","); const QVector weights = convertStringToFloatVector(parv[3]); - br_eval_fused(simmatList, parv[1], parv[2], 0, weights, 1e-6, 1e6); + br_eval_fused(simmatList, parv[1], parv[2], 0, weights, -1e6, 1e6); } else { const QStringList simmatList = QString(parv[0]).split(","); const QVector weights = convertStringToFloatVector(parv[3]); diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index b354e0936..baf378c55 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -502,16 +502,16 @@ float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QStrin lines.append(qPrintable(QString("BC,0.001,%1").arg(QString::number(result = getOperatingPoint(operatingPoints, "FAR", 0.001).TAR, 'f', 3)))); // Attempt to read template size from enrolled gallery and write to output CSV - size_t maxSize(0); - // size_t maxSize2(0); - // size_t maxSize3(0); + size_t maxSizeTotal(0); for (int i=0; i 0) qDebug("Template Size: %i bytes", (int)maxSize); + if (maxSizeTotal > 0) qDebug("Template Size: %i bytes", (int)maxSizeTotal); foreach (float FAR, QList() << 1e-1 << 1e-2 << 1e-3 << 1e-4 << 1e-5 << 1e-6) { const OperatingPoint op = getOperatingPoint(operatingPoints, "FAR", FAR); printf("TAR & Similarity @ FAR = %.0e: %.4f %.3f\n", FAR, op.TAR, op.score); From f09f1179414eeae0a09b0cf88b7f145e891d61b3 Mon Sep 17 00:00:00 2001 From: Steven Date: Fri, 6 Dec 2024 00:28:22 +0000 Subject: [PATCH 10/12] condensing number of Evaluate declarations --- openbr/core/eval.cpp | 96 ++++++++++++++++---------------------------- openbr/core/eval.h | 1 - openbr/openbr.cpp | 2 +- 3 files changed, 35 insertions(+), 64 deletions(-) diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index baf378c55..a865bd4eb 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -151,43 +151,6 @@ float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &qu return Evaluate(scores, constructMatchingMask(scores, target, query, partition), csv, QStringList(), QString(), 0); } -float Evaluate(const QString &simmat, const QString &mask, const File &csv, unsigned int matches) -{ - qDebug("Evaluating %s%s%s", - qPrintable(simmat), - mask.isEmpty() ? "" : qPrintable(" with " + mask), - csv.name.isEmpty() ? "" : qPrintable(" to " + csv)); - - // Read similarity matrix - QString target, query; - Mat scores; - if (simmat.endsWith(".mtx")) { - scores = BEE::readMatrix(simmat, &target, &query); - } else { - QScopedPointer format(Factory::make(simmat)); - scores = format->read(); - } - - // Read mask matrix - Mat truth; - if (mask.isEmpty()) { - // Use the galleries specified in the similarity matrix - if (target.isEmpty()) qFatal("Unspecified target gallery."); - if (query.isEmpty()) qFatal("Unspecified query gallery."); - - truth = constructMatchingMask(scores, TemplateList::fromGallery(target).files(), - TemplateList::fromGallery(query).files()); - } else { - File maskFile(mask); - maskFile.set("rows", scores.rows); - maskFile.set("columns", scores.cols); - QScopedPointer format(Factory::make(maskFile)); - truth = format->read(); - } - - return Evaluate(scores, truth, csv, {target}, query, matches); -} - float Evaluate(const QStringList &simmats, const QString &mask, const File &csv, unsigned int matches, const QVector &weights, float lowerBound, float upperBound) { if (simmats.size() <= 0) { @@ -199,10 +162,17 @@ float Evaluate(const QStringList &simmats, const QString &mask, const File &csv, } int n_simmats = simmats.size(); - qDebug("Evaluating %d simmats%s%s", - n_simmats, + if (n_simmats == 1) { + qDebug("Evaluating %s%s%s", + qPrintable(simmats[0]), mask.isEmpty() ? "" : qPrintable(" with " + mask), csv.name.isEmpty() ? "" : qPrintable(" to " + csv)); + } else { + qDebug("Evaluating %d simmats%s%s", + n_simmats, + mask.isEmpty() ? "" : qPrintable(" with " + mask), + csv.name.isEmpty() ? "" : qPrintable(" to " + csv)); + } QVector scores(n_simmats); QStringList targets; @@ -239,31 +209,33 @@ float Evaluate(const QStringList &simmats, const QString &mask, const File &csv, truth = format->read(); } - float count_total = 0; - float count_fused = 0; - for (int i = 0; i < scores[0].rows; i++) { - for (int j = 0; j < scores[0].cols; j++) { - const BEE::MaskValue mask_val = truth.at(i,j); - if (mask_val == BEE::DontCare) continue; - - BEE::SimmatValue simmat_val = scores[0].at(i,j); - if (simmat_val != simmat_val) continue; - - if (simmat_val >= lowerBound && simmat_val <= upperBound) { - simmat_val *= weights[0]; - for (int k = 1; k < n_simmats; k++) { - simmat_val += weights[k]*scores[k].at(i,j); - } - scores[0].at(i,j) = simmat_val; - count_fused += 1; + if (n_simmats > 1) { + float count_total = 0; + float count_fused = 0; + for (int i = 0; i < scores[0].rows; i++) { + for (int j = 0; j < scores[0].cols; j++) { + const BEE::MaskValue mask_val = truth.at(i,j); + if (mask_val == BEE::DontCare) continue; + + BEE::SimmatValue simmat_val = scores[0].at(i,j); + if (simmat_val != simmat_val) continue; + + if (simmat_val >= lowerBound && simmat_val <= upperBound) { + simmat_val *= weights[0]; + for (int k = 1; k < n_simmats; k++) { + simmat_val += weights[k]*scores[k].at(i,j); + } + scores[0].at(i,j) = simmat_val; + count_fused += 1; + } + count_total += 1; } - count_total += 1; } + qDebug("fused %f percent of comparisons w/ scores falling between [%f, %f]", + count_fused/count_total*100, + lowerBound, + upperBound); } - qDebug("fused %f percent of comparisons w/ scores falling between [%f, %f]", - count_fused/count_total*100, - lowerBound, - upperBound); return Evaluate(scores[0], truth, csv, targets, query, matches); } @@ -554,7 +526,7 @@ float Evaluate(const Mat &simmat, const Mat &mask, const File &csv, const QStrin void assertEval(const QString &simmat, const QString &mask, float accuracy) { - float result = Evaluate(simmat, mask, "", 0); + float result = Evaluate({simmat}, mask, "", 0, QVector(), -1e6, 1e6); // Round result to nearest thousandth for comparison against input accuracy. Input is expected to be from previous // results of br -eval. result = floor(result*1000+0.5)/1000; diff --git a/openbr/core/eval.h b/openbr/core/eval.h index e6a213b60..58c31d2d2 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -75,7 +75,6 @@ namespace br { - float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 float Evaluate(const QStringList &simmats, const QString &mask = "", const File &csv = "", unsigned int matches = 0, const QVector &weights = QVector(), float lowerBound = -1e6, float upperBound = 1e6); // Returns TAR @ FAR = 0.001 float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0); float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QStringList &targets = QStringList(), const QString &query = "", unsigned int matches = 0); diff --git a/openbr/openbr.cpp b/openbr/openbr.cpp index 4547a65c4..1eac02250 100644 --- a/openbr/openbr.cpp +++ b/openbr/openbr.cpp @@ -103,7 +103,7 @@ void br_project(const char *input, const char *gallery) float br_eval(const char *simmat, const char *mask, const char *csv, int matches) { - return Evaluate(simmat, mask, csv, matches); + return Evaluate({simmat}, mask, csv, matches, QVector(), -1e6, 1e6); } float br_eval_fused(const QStringList &simmats, const char *mask, const char *csv, int matches, const QVector &weights, float lowerBound, float upperBound) From 00008b8a7407b7c67d6f5ab20cb858686fbf7c49 Mon Sep 17 00:00:00 2001 From: Steven Date: Fri, 6 Dec 2024 00:39:38 +0000 Subject: [PATCH 11/12] removing commented out code --- openbr/core/eval.h | 50 ---------------------------------------------- 1 file changed, 50 deletions(-) diff --git a/openbr/core/eval.h b/openbr/core/eval.h index 58c31d2d2..bf67e7f61 100644 --- a/openbr/core/eval.h +++ b/openbr/core/eval.h @@ -1,53 +1,3 @@ -// /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * -// * Copyright 2012 The MITRE Corporation * -// * * -// * Licensed under the Apache License, Version 2.0 (the "License"); * -// * you may not use this file except in compliance with the License. * -// * You may obtain a copy of the License at * -// * * -// * http://www.apache.org/licenses/LICENSE-2.0 * -// * * -// * Unless required by applicable law or agreed to in writing, software * -// * distributed under the License is distributed on an "AS IS" BASIS, * -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * -// * See the License for the specific language governing permissions and * -// * limitations under the License. * -// * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -// #ifndef BR_EVAL_H -// #define BR_EVAL_H - -// #include -// #include -// #include "openbr/openbr_plugin.h" - -// namespace br -// { -// float Evaluate(const QString &simmat, const QString &mask = "", const File &csv = "", unsigned int matches = 0); // Returns TAR @ FAR = 0.001 -// float Evaluate(const QString &simmat, const QString &simmat2, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.7, float w2=0.3, float lowerBound=-1e6, float upperBound=1e6); // Returns TAR @ FAR = 0.001 -// float Evaluate(const QString &simmat, const QString &simmat2, const QString &simmat3, const QString &mask = "", const File &csv = "", unsigned int matches = 0, float w1=0.35, float w2=0.35, float w3=0.3, float lowerBound=-1e6, float upperBound=1e6); // Returns TAR @ FAR = 0.001 -// float Evaluate(const cv::Mat &scores, const FileList &target, const FileList &query, const File &csv = "", int parition = 0); -// float Evaluate(const cv::Mat &scores, const cv::Mat &masks, const File &csv = "", const QString &target = "", const QString &query = "", unsigned int matches = 0, const QString &target2 = "", const QString &target3 = ""); -// void assertEval(const QString &simmat, const QString &mask, float accuracy); // Check to see if -eval achieves a given TAR @ FAR = 0.001 -// float InplaceEval(const QString &simmat, const QString &mask, const QString &csv); - -// void EvalClassification(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = ""); -// float EvalDetection(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", bool normalize = false, int minSize = 0, int maxSize = 0, float relativeMinSize = 0, const QString &label = "", const float true_positive_threshold = 0.5f); // Return average overlap -// float EvalLandmarking(const QString &predictedGallery, const QString &truthGallery, const QString &csv = "", int normalizationIndexA = 0, int normalizationIndexB = 1, int sampleIndex = 0, int totalExamples = 5); // Return average error -// void EvalRegression(const QString &predictedGallery, const QString &truthGallery, QString predictedProperty = "", QString truthProperty = "", bool generatePlots = true); -// void EvalKNN(const QString &knnGraph, const QString &knnTruth, const QString &csv = ""); -// void EvalEER(const QString &predictedXML, const QString gt_property = "", const QString distribution_property = "", const QString &csv = ""); -// struct Candidate -// { -// size_t index; -// float similarity; - -// Candidate() {} -// Candidate(size_t index_, float similarity_) -// : index(index_), similarity(similarity_) {} -// }; -// } - // #endif // BR_EVAL_H /* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * From cc2ad53057469d21d52a6495d459603ee1464f3b Mon Sep 17 00:00:00 2001 From: Steven Date: Fri, 6 Dec 2024 18:34:29 +0000 Subject: [PATCH 12/12] updating percent fusion counters to double to avoid overflow --- openbr/core/eval.cpp | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openbr/core/eval.cpp b/openbr/core/eval.cpp index a865bd4eb..515dc764e 100755 --- a/openbr/core/eval.cpp +++ b/openbr/core/eval.cpp @@ -210,8 +210,8 @@ float Evaluate(const QStringList &simmats, const QString &mask, const File &csv, } if (n_simmats > 1) { - float count_total = 0; - float count_fused = 0; + double count_total = 0; + double count_fused = 0; for (int i = 0; i < scores[0].rows; i++) { for (int j = 0; j < scores[0].cols; j++) { const BEE::MaskValue mask_val = truth.at(i,j); @@ -222,16 +222,15 @@ float Evaluate(const QStringList &simmats, const QString &mask, const File &csv, if (simmat_val >= lowerBound && simmat_val <= upperBound) { simmat_val *= weights[0]; - for (int k = 1; k < n_simmats; k++) { - simmat_val += weights[k]*scores[k].at(i,j); - } + for (int k = 1; k < n_simmats; k++) + simmat_val += weights[k]*scores[k].at(i,j); scores[0].at(i,j) = simmat_val; - count_fused += 1; + count_fused++; } - count_total += 1; + count_total++; } - } - qDebug("fused %f percent of comparisons w/ scores falling between [%f, %f]", + } + qDebug("fused %lf percent of comparisons w/ scores falling between [%f, %f]", count_fused/count_total*100, lowerBound, upperBound);