Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Record the SMT input size for each split #938

Open
wants to merge 9 commits into
base: master
Choose a base branch
from
6 changes: 5 additions & 1 deletion Source/Core/Xml.cs
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ public void WriteEndMethod(string outcome, DateTime endTime, TimeSpan elapsed, i
}

public void WriteSplit(int splitNum, int iteration, IEnumerable<AssertCmd> asserts, DateTime startTime,
string outcome, TimeSpan elapsed, int? resourceCount)
string outcome, TimeSpan elapsed, int? resourceCount, int? smtInputSize)
{
Contract.Requires(splitNum > 0);
Contract.Requires(outcome != null);
Expand Down Expand Up @@ -145,6 +145,10 @@ public void WriteSplit(int splitNum, int iteration, IEnumerable<AssertCmd> asser
{
wr.WriteAttributeString("resourceCount", resourceCount.ToString());
}
if (smtInputSize is not null)
{
wr.WriteAttributeString("smtInputSize", smtInputSize.ToString());
}
wr.WriteEndElement(); // conclusion

wr.WriteEndElement(); // assertionBatch
Expand Down
2 changes: 1 addition & 1 deletion Source/ExecutionEngine/ImplementationRunResult.cs
Original file line number Diff line number Diff line change
Expand Up @@ -78,7 +78,7 @@ public void ProcessXml(ExecutionEngine engine) {

foreach (var vcResult in RunResults.OrderBy(s => (vcNum: s.VcNum, iteration: s.Iteration))) {
engine.Options.XmlSink.WriteSplit(vcResult.VcNum, vcResult.Iteration, vcResult.Asserts, vcResult.StartTime,
vcResult.Outcome.ToString().ToLowerInvariant(), vcResult.RunTime, vcResult.ResourceCount);
vcResult.Outcome.ToString().ToLowerInvariant(), vcResult.RunTime, vcResult.ResourceCount, vcResult.SmtInputSize);
}

engine.Options.XmlSink.WriteEndMethod(VcOutcome.ToString().ToLowerInvariant(),
Expand Down
3 changes: 3 additions & 0 deletions Source/Provers/SMTLib/ProverInterface.cs
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Diagnostics.Contracts;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Microsoft.Boogie.SMTLib;
Expand All @@ -21,6 +22,8 @@ public enum SolverOutcome

public abstract class ProverInterface
{
public int SentSize { get; protected set; }

public static ProverInterface CreateProver(SMTLibOptions libOptions, Program prog,
string /*?*/ logFilePath, bool appendLogFile, uint timeout,
int taskID = -1)
Expand Down
7 changes: 5 additions & 2 deletions Source/Provers/SMTLib/SMTLibBatchTheoremProver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -71,9 +71,9 @@ public override async Task<SolverOutcome> Check(string descriptiveName, VCExpr v
SetupProcess();
checkSatSent = false;
FullReset(gen);

if (options.LogFilename != null && currentLogFile == null)
{

currentLogFile = OpenOutputFile(descriptiveName);
await currentLogFile.WriteAsync(common.ToString());
}
Expand Down Expand Up @@ -123,6 +123,7 @@ public override void FullReset(VCExpressionGenerator generator)
AxiomsAreSetup = false;
DeclCollector.Reset();
NamedAssumes.Clear();
SentSize = 0;
}

private Task<IReadOnlyList<SExpr>> SendRequestsAndClose(IReadOnlyList<string> requests, CancellationToken cancellationToken) {
Expand Down Expand Up @@ -284,7 +285,9 @@ protected override void Send(string s, bool isCommon)
// Boogie emits comments after the solver has responded. In batch
// mode, sending these to the solver is problematic. But they'll
// still get sent to the log below.
if (Process != null && !checkSatSent) {
if (Process != null && !checkSatSent)
{
SentSize += s.Length;
Process.Send(s);
}

Expand Down
11 changes: 9 additions & 2 deletions Source/Provers/SMTLib/SMTLibInteractiveTheoremProver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,9 @@ private void PossiblyRestart()
if (Process != null && processNeedsRestart) {
processNeedsRestart = false;
SetupProcess();
Process.Send(common.ToString());
var c = common.ToString();
SentSize += c.Length;
Process.Send(c);
}
}

Expand Down Expand Up @@ -124,6 +126,7 @@ public override async Task<SolverOutcome> Check(string descriptiveName, VCExpr v
}

var result = await CheckSat(cancellationToken, errorLimit);
SentSize = 0;
SendThisVC("(pop 1)");

return result;
Expand All @@ -142,10 +145,11 @@ public override async Task Reset(VCExpressionGenerator generator)
if (0 < common.Length)
{
var c = common.ToString();
SentSize += c.Length;
Process.Send(c);
if (currentLogFile != null)
{
currentLogFile.WriteLine(c);
await currentLogFile.WriteLineAsync(c);
}
}

Expand Down Expand Up @@ -264,6 +268,7 @@ public async Task<SolverOutcome> CheckSat(CancellationToken cancellationToken,
{
if (popLater)
{
SentSize = 0;
SendThisVC("(pop 1)");
}
}
Expand Down Expand Up @@ -301,6 +306,7 @@ private T WrapInPushPop<T>(ref bool popLater, Func<T> action)
{
if (popLater)
{
SentSize = 0;
SendThisVC("(pop 1)");
}
SendThisVC("(push 1)");
Expand Down Expand Up @@ -647,6 +653,7 @@ protected override void Send(string s, bool isCommon)

if (Process != null)
{
SentSize += s.Length;
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It would be lovely if this could only be done from a wrapper around Send, to ensure that it's always incremented. It would be even nicer if the reset to size 0 could be encapsulated somewhere, too, though I remember from working on this code that there isn't a clear place to put it at the moment.

Process.Send(s);
}

Expand Down
1 change: 1 addition & 0 deletions Source/Provers/SMTLib/SMTLibProcessTheoremProver.cs
Original file line number Diff line number Diff line change
Expand Up @@ -1111,6 +1111,7 @@ public override void PushVCExpression(VCExpr vc)

public override void Pop()
{
SentSize = 0;
SendThisVC("(pop 1)");
DeclCollector.Pop();
}
Expand Down
2 changes: 2 additions & 0 deletions Source/VCGeneration/Checker.cs
Original file line number Diff line number Diff line change
Expand Up @@ -259,6 +259,8 @@ public TimeSpan ProverRunTime
get { return proverRunTime; }
}

public int SmtInputSize => thmProver.SentSize;

public int GetProverResourceCount()
{
return thmProver.GetRCount();
Expand Down
12 changes: 6 additions & 6 deletions Source/VCGeneration/Split.cs
Original file line number Diff line number Diff line change
Expand Up @@ -39,16 +39,14 @@ public IReadOnlyList<Declaration> PrunedDeclarations {
}
}

readonly Dictionary<Block /*!*/, BlockStats /*!*/> /*!*/
stats = new Dictionary<Block /*!*/, BlockStats /*!*/>();
readonly Dictionary<Block /*!*/, BlockStats /*!*/> /*!*/ stats = new();

static int currentId = -1;

Block splitBlock;
bool assertToAssume;

List<Block /*!*/> /*!*/
assumizedBranches = new List<Block /*!*/>();
List<Block /*!*/> /*!*/ assumizedBranches = new();

double score;
bool scoreComputed;
Expand All @@ -58,8 +56,7 @@ public IReadOnlyList<Declaration> PrunedDeclarations {

public Dictionary<TransferCmd, ReturnCmd> GotoCmdOrigins { get; }

public readonly VerificationConditionGenerator /*!*/
parent;
public readonly VerificationConditionGenerator /*!*/ parent;

public Implementation /*!*/ Implementation => Run.Implementation;

Expand All @@ -78,6 +75,7 @@ public readonly VerificationConditionGenerator /*!*/
// async interface
public int SplitIndex { get; set; }
public VerificationConditionGenerator.ErrorReporter reporter;
public int SmtInputSize { get; set; }

public Split(VCGenOptions options, List<Block /*!*/> /*!*/ blocks,
Dictionary<TransferCmd, ReturnCmd> /*!*/ gotoCmdOrigins,
Expand Down Expand Up @@ -915,6 +913,7 @@ public VerificationRunResult ReadOutcome(int iteration, Checker checker, Verifie
Asserts: Asserts,
CoveredElements: CoveredElements,
ResourceCount: resourceCount,
SmtInputSize,
SolverUsed: (Options as SMTLibSolverOptions)?.Solver);
callback.OnVCResult(result);

Expand Down Expand Up @@ -981,6 +980,7 @@ public async Task BeginCheck(TextWriter traceWriter, Checker checker, VerifierCa
checker.TheoremProver.SetAdditionalSmtOptions(Implementation.GetExtraSMTOptions()
.Select(kv => new OptionValue(kv.Key, kv.Value)));
await checker.BeginCheck(Description, vc, reporter, timeout, rlimit, cancellationToken);
SmtInputSize = checker.SmtInputSize;
}

public string Description
Expand Down
1 change: 1 addition & 0 deletions Source/VCGeneration/VerificationRunResult.cs
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ public record VerificationRunResult
List<AssertCmd> Asserts,
IEnumerable<TrackedNodeComponent> CoveredElements,
int ResourceCount,
int SmtInputSize,
SolverKind? SolverUsed
) {
public void ComputePerAssertOutcomes(out Dictionary<AssertCmd, SolverOutcome> perAssertOutcome,
Expand Down
8 changes: 4 additions & 4 deletions Test/commandline/xml.bpl
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
// RUN: %parallel-boogie -randomSeed:0 -xml:"%t-1.xml" "%s"
// RUN: %parallel-boogie -randomSeed:0 -xml:"%t-2.xml" "%s"
// RUN: %parallel-boogie -randomSeed:0 -proverLog:%t-1.smt -xml:"%t-1.xml" "%s"
// RUN: %parallel-boogie -randomSeed:0 -proverLog:%t-2.smt -xml:"%t-2.xml" "%s"
// RUN: grep -Eo "resourceCount=\"[0-9]+\"" "%t-1.xml" | sort -g > "%t-res1"
// RUN: grep -Eo "resourceCount=\"[0-9]+\"" "%t-2.xml" | sort -g > "%t-res2"
// RUN: diff "%t-res1" "%t-res2"
Expand All @@ -11,11 +11,11 @@
// CHECK: \<method name="ExampleWithSplits" startTime=".*"\>
// CHECK: \<assertionBatch number="1" iteration="0" startTime=".*"\>
// CHECK: \<assertion file="xml.bpl" line="25" column="3" /\>
// CHECK: \<conclusion duration=".*" outcome="valid" resourceCount=".*" /\>
// CHECK: \<conclusion duration=".*" outcome="valid" resourceCount=".*" smtInputSize="865" /\>
// CHECK: \</assertionBatch\>
// CHECK: \<assertionBatch number="2" iteration="0" startTime=".*"\>
// CHECK: \<assertion file="xml.bpl" line="27" column="3" /\>
// CHECK: \<conclusion duration=".*" outcome="valid" resourceCount=".*" /\>
// CHECK: \<conclusion duration=".*" outcome="valid" resourceCount=".*" smtInputSize="881" /\>
// CHECK: \</assertionBatch\>
// CHECK: \<conclusion endTime=".*" duration=".*" resourceCount=".*" outcome="correct" /\>
// CHECK: \</method\>
Expand Down
Loading