Skip to content

[TS] Measure paired source coverage and search cost #416

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
CaelmBleidd wants to merge 1 commit into
base: caelmbleidd/iccq-coverage-v2
Choose a base branch
from
Open

[TS] Measure paired source coverage and search cost #416

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 94 additions & 3 deletions usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageCell.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,26 @@
import java.nio.file.Path
import java.nio.file.StandardCopyOption
import java.nio.file.StandardOpenOption
import java.security.MessageDigest
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.TimeSource

@Serializable
internal data class CallsCoverageUniverseIdentity(
val toolRevision: String,
val nativeFrontendRevision: String,
val sourceManifest: String,
val sourceIdentity: CallsCoverageSourceIdentity,
val projectIndex: Int,
val functionIndex: Int,
val sourceFile: String,
)

@Serializable
internal data class CallsCoverageUniverseRecord(
val identity: CallsCoverageUniverseIdentity,
val universe: CallsCoverageUniverse,
)

@Serializable
internal data class CallsCoverageCellIdentity(
Expand All @@ -25,6 +44,7 @@
val budgetMillis: Long,
val solverLimitMillis: Long,
val candidateCap: Int,
val coverageUniverseSha256: String,
)

@Serializable
Expand All @@ -40,6 +60,7 @@
val emittedCandidates: Int,
val replayedCandidates: Int,
val coveredStatements: Int,
val totalStatements: Int,
val coveredIfArms: Int,
)

Expand All @@ -49,16 +70,59 @@
val search: CallsCoverageSearchResult,
val replayErrors: Int,
val completionMismatches: Int,
val replayElapsedMillis: Long,
val cellElapsedMillis: Long,
val checkpoints: List<CallsCoverageCheckpoint>,
)

internal fun runCoverageUniverse(
args: List<String>,
runtimeToolRevision: String = CallsBuildIdentity.toolRevision,
runtimeNativeFrontendRevision: String = CallsBuildIdentity.nativeFrontendRevision,
probeFunction: (CallsCoverageSource) -> CallsCoverageUniverse = { source ->
OriginalTypeScriptCoverageReplayer(
sourceRoot = source.sourceRoot,
function = source.function,
).use { replayer -> replayer.probeUniverse(timeoutMillis = REPLAY_TIMEOUT_MILLIS) }
},
) {
require(args.size == 4) { "coverage-universe requires 4 arguments" }
val projectIndex = args[1].toInt()
val functionIndex = args[2].toInt()
val source = CallsCoverageSource.load(Path.of(args[0]), projectIndex, functionIndex)
val output = Path.of(args[3]).toAbsolutePath().normalize()
require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" }

Files.createDirectories(output)
require(Files.list(output).use { entries -> entries.findAny().isEmpty }) {
"Coverage universe output directory is not empty"
}
val identity = CallsCoverageUniverseIdentity(
toolRevision = runtimeToolRevision,
nativeFrontendRevision = runtimeNativeFrontendRevision,
sourceManifest = source.manifestPath.toString(),
sourceIdentity = source.identity,
projectIndex = projectIndex,
functionIndex = functionIndex,
sourceFile = source.function.sourceFile,
)
require(Regex("[0-9a-f]{40}").matches(identity.toolRevision)) { "Coverage build must be clean" }
source.verifyUnchanged()
val universe = probeFunction(source)
source.verifyUnchanged()
val record = CallsCoverageUniverseRecord(identity = identity, universe = universe)
atomicWrite(output.resolve("universe.json"), encode(record))
println(encode(record))
}

internal fun runCoverageCell(
args: List<String>,
runtimeToolRevision: String = CallsBuildIdentity.toolRevision,
runtimeNativeFrontendRevision: String = CallsBuildIdentity.nativeFrontendRevision,
searchFunction: (CallsCoverageSearchRequest) -> CallsCoverageSearchResult = CurrentTsCallsCoverageEngine()::search,
) {
require(args.size == 9) { "coverage-cell requires 9 arguments" }
require(args.size == 10) { "coverage-cell requires 10 arguments" }
val cellStarted = TimeSource.Monotonic.markNow()
val projectIndex = args[1].toInt()
val functionIndex = args[2].toInt()
val source = CallsCoverageSource.load(Path.of(args[0]), projectIndex, functionIndex)
Expand All @@ -70,11 +134,28 @@
val budgetMillis = args[6].toLong()
val solverLimitMillis = args[7].toLong()
val candidateCap = args[8].toInt()
val universeBytes = Files.readAllBytes(Path.of(args[9]).toRealPath())
val universeRecord = CallsExperimentJson.json.decodeFromString<CallsCoverageUniverseRecord>(
universeBytes.toString(Charsets.UTF_8),
)
val project = source.project
val function = source.function
require(seed in manifest.seeds) { "Seed is outside the frozen source manifest" }
require(budgetMillis > 0 && solverLimitMillis in 1..budgetMillis && candidateCap > 0)
require(System.getenv("ETS_FRONTEND_SCRIPT") == null) { "ETS_FRONTEND_SCRIPT must be unset" }
require(universeRecord.identity == CallsCoverageUniverseIdentity(
toolRevision = runtimeToolRevision,
nativeFrontendRevision = runtimeNativeFrontendRevision,
sourceManifest = manifestPath.toString(),
sourceIdentity = source.identity,
projectIndex = projectIndex,
functionIndex = functionIndex,
sourceFile = function.sourceFile,
)) { "Coverage universe identity differs" }
val universe = universeRecord.universe
require(function.sourceFile in universe.sourceFiles && universe.supportedStatementKeys.isNotEmpty()) {
"Coverage universe does not include the entry source statements"
}

val identity = CallsCoverageCellIdentity(
schemaVersion = 2,
Expand All @@ -89,6 +170,8 @@
budgetMillis = budgetMillis,
solverLimitMillis = solverLimitMillis,
candidateCap = candidateCap,
coverageUniverseSha256 = MessageDigest.getInstance("SHA-256").digest(universeBytes)
.joinToString("") { byte -> "%02x".format(byte.toInt() and 0xff) },
)
require(Regex("[0-9a-f]{40}").matches(identity.toolRevision)) { "Coverage build must be clean" }

Expand Down Expand Up @@ -166,12 +249,18 @@
search.candidates[index].emittedAtMillis <= checkpoint
}
val replayed = indices.mapNotNull { index -> events[index].replay }
val coveredStatements = replayed.flatMap { it.coveredStatementKeys }.toSet()
val coveredIfArms = replayed.flatMap { it.coveredIfArmKeys }.toSet()
require(universe.supportedStatementKeys.containsAll(coveredStatements)) {
"Replayed statements extend beyond the frozen coverage universe"
}
CallsCoverageCheckpoint(
searchBudgetMillis = checkpoint,
emittedCandidates = indices.size,
replayedCandidates = replayed.size,
coveredStatements = replayed.flatMap { it.coveredStatementKeys }.toSet().size,
coveredIfArms = replayed.flatMap { it.coveredIfArmKeys }.toSet().size,
coveredStatements = coveredStatements.size,
totalStatements = universe.supportedStatementKeys.size,
coveredIfArms = coveredIfArms.size,
)
}
val result = CallsCoverageCellResult(
Expand All @@ -182,6 +271,8 @@
events[index].replay?.completion != null &&
events[index].replay?.completion != search.candidates[index].completion
},
replayElapsedMillis = events.sumOf { event -> event.replay?.replayElapsedMillis ?: 0L },
cellElapsedMillis = cellStarted.elapsedNow().inWholeMilliseconds,
checkpoints = checkpoints,
)
atomicWrite(output.resolve("result.json"), encode(result))
Expand Down
64 changes: 64 additions & 0 deletions usvm-ts-calls/src/main/kotlin/org/usvm/ts/calls/CallsCoverageReplay.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,14 @@ internal data class CallsCoverageReplayResult(
val replayElapsedMillis: Long,
)

@Serializable
internal data class CallsCoverageUniverse(
val supportedStatementKeys: Set<String>,
val sourceFiles: Set<String>,
val diagnostics: List<String>,
val probeElapsedMillis: Long,
)

/** Replays one generated input in a fresh Node.js process and checks its exact transport value. */
internal class OriginalTypeScriptCoverageReplayer(
sourceRoot: Path,
Expand Down Expand Up @@ -88,6 +96,58 @@ internal class OriginalTypeScriptCoverageReplayer(
}
}

fun probeUniverse(timeoutMillis: Long): CallsCoverageUniverse {
require(timeoutMillis > 0)

val started = TimeSource.Monotonic.markNow()
val result = backend.run(
property = PropertyDefinition(
id = PropertyId("calls.coverage.universe"),
inputs = listOf(PropertyInput(name = "__usvmCoverageSentinel", domain = BooleanDomain)),
predicate = TypeScriptEntryPoint(module = wrapperName, exportName = "coverageUniverseProbe"),
),
configuration = PropertyRunConfiguration(
seed = 0,
numRuns = 1,
timeoutMillis = timeoutMillis,
examples = listOf(listOf(JsConcreteValue.Boolean(true))),
coverageRequest = PropertyCoverageRequest(
scopes = setOf(CoverageScope.SOURCE_UNDER_TEST),
includePatterns = allowedFiles.sorted(),
includeUnexecutedSources = true,
),
),
)
val elapsedMillis = started.elapsedNow().inWholeMilliseconds
require(result.status == PropertyRunStatus.SUCCESS && result.numRuns == 1) {
"Coverage universe probe did not complete once: $result"
}

val coverage = requireNotNull(result.coverage) { "Coverage universe probe produced no c8 artifact" }
val statements = linkedSetOf<String>()
val sourceFiles = linkedSetOf<String>()
coverage.files.forEach { file ->
val relative = relativeSourcePath(file.path) ?: return@forEach
if (relative !in allowedFiles) return@forEach

sourceFiles += relative
file.statements.forEach { statement ->
statements += "$relative:statement:${statement.location.stableKey()}"
}
}
require(sourceFiles == allowedFiles) {
"Coverage universe omitted local source files: ${allowedFiles - sourceFiles}"
}
require(statements.isNotEmpty()) { "Coverage universe has no source statements" }

return CallsCoverageUniverse(
supportedStatementKeys = statements,
sourceFiles = sourceFiles,
diagnostics = coverage.diagnostics.map { diagnostic -> "${diagnostic.code}: ${diagnostic.message}" },
probeElapsedMillis = elapsedMillis,
)
}

fun replay(inputs: List<JsConcreteValue>, timeoutMillis: Long): CallsCoverageReplayResult {
require(timeoutMillis > 0)
require(inputs.size == function.inputs.size)
Expand Down Expand Up @@ -252,5 +312,9 @@ private fun coverageWrapper(

return true;
}

export function coverageUniverseProbe(_sentinel: boolean): boolean {
return true;
}
""".trimIndent() + "\n"
}
Loading
Loading