diff --git a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 index d1f6d1d397..e175228b26 100644 --- a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 +++ b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaLexer.g4 @@ -76,6 +76,8 @@ CharacterClassEscape : SLASH [dDsSwWvVhH] ; +EQUAL : '='; +LESS_THAN : '<'; CARET : '^'; DOLLAR : '$'; SLASH : '\\'; @@ -96,7 +98,7 @@ COLON : ':'; BaseChar // practically all chars but the ones used for control and digits - : ~[0-9:,^$\\.*+?()[\]{}|-] + : ~[0-9:,^$\\.<*+?()[\]{}|=-] ; fragment OctalEscapeSequence diff --git a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 index 6a0a9425ed..27f14c38c0 100644 --- a/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 +++ b/core/src/main/antlr4/org/evomaster/core/parser/RegexJavaParser.g4 @@ -52,7 +52,8 @@ assertion //TODO //// | '\\' 'b' //// | '\\' 'B' -//// | '(' '?' '=' disjunction ')' + | PAREN_open QUESTION EQUAL disjunction PAREN_close // lookahead (?=...) + | PAREN_open QUESTION LESS_THAN EQUAL disjunction PAREN_close // lookbehind (?<=...) //// | '(' '?' '!' disjunction ')' ; @@ -116,7 +117,7 @@ patternCharacter // These are also allowed as literals when no matching pair exists | BRACE_close | BRACKET_close - | COLON + | COLON | EQUAL | LESS_THAN | DOUBLE_AMPERSAND // char class intersection not supported by default in JS, only supported if "v" flag is turned on. ; @@ -168,7 +169,7 @@ classAtomNoDash | DecimalDigit | COMMA | CARET | DOLLAR | DOT | STAR | PLUS | QUESTION | PAREN_open | PAREN_close | BRACKET_open | BRACE_open | BRACE_close | OR - | COLON + | COLON | EQUAL | LESS_THAN // should be interpreted literally: // As they are lexer tokens, these character sequences are captured as such. In particular these require some extra // steps to interpret them correctly given the context. diff --git a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt index aa01cfb3bb..32617e49e2 100644 --- a/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt +++ b/core/src/main/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitor.kt @@ -11,7 +11,7 @@ private const val EOF_TOKEN = "" /** * Created by arcuri82 on 11-Sep-19. */ -class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : RegexJavaParserBaseVisitor(){ +class GeneRegexJavaVisitor(val sourceRegex: String, val externalRegexFlags: RegexFlags = RegexFlags()) : RegexJavaParserBaseVisitor(){ private val hexEscapePrefixes = setOf('x', 'u') @@ -60,6 +60,8 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege */ private var currentFlags = externalRegexFlags + private var hasAssertions = false + /** * Builds DisjunctionListRxGenes from a disjunction context, returns null if disjunction is unsatisfiable. */ @@ -87,6 +89,23 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege return disjList } + /** + * Walks up [ctx]'s ancestry towards the top-level pattern, searching for one of + * the currently-unsupported ways an assertion's ancestry can appear (nested). + * Returns true if it reaches the top-level pattern without hitting either, false otherwise. + */ + private fun isAssertionNested(ctx: RegexJavaParser.AssertionContext): Boolean { + var current = ctx.parent + while (current != null && current !is RegexJavaParser.PatternContext) { + if (current is RegexJavaParser.AssertionContext + || (current is RegexJavaParser.AtomContext && current.disjunction() != null)) { + return true + } + current = current.parent + } + return false + } + override fun visitPattern(ctx: RegexJavaParser.PatternContext): VisitResult { val res = ctx.disjunction().accept(this) @@ -107,8 +126,10 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege val gene = RegexGene( "regex", disjList, - text.substring(0, text.length - EOF_TOKEN.length), - RegexType.JVM + sourceRegex, + RegexType.JVM, + externalRegexFlags = externalRegexFlags, + hasAssertions = hasAssertions ) return VisitResult(gene) @@ -231,7 +252,23 @@ class GeneRegexJavaVisitor(externalRegexFlags: RegexFlags = RegexFlags()) : Rege val res = VisitResult() if(ctx.assertion() != null){ - res.data = ctx.assertion().text + val assertionCtx = ctx.assertion() + if (assertionCtx.CARET() != null || assertionCtx.DOLLAR() != null) { + res.data = ctx.assertion().text + } else { + require(!isAssertionNested(ctx.assertion())){ + "Nested assertions are not currently supported." + } + val innerDisjList = buildDisjunctionList(assertionCtx.disjunction()) + val assertionType = if (assertionCtx.LESS_THAN() != null) { + AssertionType.LOOKBEHIND + } else { + AssertionType.LOOKAHEAD + } + val assertionGene = AssertionRxGene(innerDisjList, assertionType) + hasAssertions = true + res.genes.add(assertionGene) + } return res } diff --git a/core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt b/core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt index 0835531389..a1a522ec0a 100644 --- a/core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt +++ b/core/src/main/kotlin/org/evomaster/core/parser/RegexHandler.kt @@ -44,7 +44,7 @@ object RegexHandler { val pattern = parser.pattern() - val res = GeneRegexJavaVisitor(externalRegexFlags).visit(pattern) + val res = GeneRegexJavaVisitor(regex, externalRegexFlags).visit(pattern) val gene= res.genes.first() as RegexGene cacheJVM[key] = gene.copy() as RegexGene diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt index 847937f9cb..3bbf1e0bcb 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AnyCharacterRxGene.kt @@ -131,4 +131,38 @@ class AnyCharacterRxGene( return true } + /** + * 1 if [value]'s first character is within [validRanges] (i.e. one `.` itself could + * render, given the current flags), else 0. + * @see [RxAbsorbable.absorbableCount] + */ + override fun absorbableCount(value: String): Int { + if (value.isEmpty()) { + return 0 + } + return if (validRanges.contains(value[0])) { + 1 + } else { + 0 + } + } + + /** Always false: `.` always renders exactly one character. + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = false + + /** + * Forces [value]'s first character onto this gene if `.` could render it; mirrors + * [absorbableCount], so it never mutates when [absorbableCount] would return 0. + * @see [RxAbsorbable.tryForce] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + val n = absorbableCount(value) + if (n == 1) { + this.value = value[0] + } + return n + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt new file mode 100644 index 0000000000..3082b511b5 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/AssertionRxGene.kt @@ -0,0 +1,147 @@ +package org.evomaster.core.search.gene.regex + +import org.evomaster.core.output.OutputFormat +import org.evomaster.core.search.gene.Gene +import org.evomaster.core.search.gene.root.CompositeFixedGene +import org.evomaster.core.search.gene.utils.GeneUtils +import org.evomaster.core.search.service.AdaptiveParameterControl +import org.evomaster.core.search.service.Randomness +import org.evomaster.core.search.service.mutator.MutationWeightControl +import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo +import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy + +/** + * Distinguishes which direction an [AssertionRxGene] forces a candidate during repair. + */ +enum class AssertionType { LOOKAHEAD, LOOKBEHIND } + +/** + * Represents a zero-width assertion in the regex gene tree. + * + * This gene is placed in the tree at the position where the assertion appeared in the + * source regex. It produces no characters ([getValueAsPrintableString] always returns + * ""). + * + * Repair is triggered from [DisjunctionRxGene.attemptAssertionRepair], invoked by + * [RegexGene.randomize] after the disjunction's own sampled value is checked against + * the source pattern and found not to match. [assertionType] determines which side of + * the enclosing disjunction's terms the candidate gets forced onto. This gene's own + * methods stay direction-agnostic. + */ +class AssertionRxGene( + /** + * The assertion's inner disjunction gene, can be null as the disjunction can be unsatisfiable, + * in that case [innerGene] is null. + */ + val innerGene: DisjunctionListRxGene?, + val assertionType: AssertionType +) : RxTerm, CompositeFixedGene("assertion:${assertionType.name}", listOfNotNull(innerGene)) { + + /** + * To handle null [innerGene], in which case the assertion is unsatisfiable. + */ + override fun canBeChildless() = true + + override fun checkForLocallyValidIgnoringChildren(): Boolean = true + + override fun isUnsatisfiable(): Boolean = innerGene == null + + override fun isMutable(): Boolean = innerGene?.isMutable() ?: false + + override fun copyContent(): Gene { + val copy = AssertionRxGene(innerGene?.copy() as? DisjunctionListRxGene, assertionType) + copy.name = this.name + return copy + } + + override fun randomize(randomness: Randomness, tryToForceNewValue: Boolean) { + innerGene?.randomize(randomness, tryToForceNewValue) + } + + override fun customShouldApplyShallowMutation( + randomness: Randomness, + selectionStrategy: SubsetGeneMutationSelectionStrategy, + enableAdaptiveGeneMutation: Boolean, + additionalGeneMutationInfo: AdditionalGeneMutationInfo? + ): Boolean { + return false + } + + override fun shallowMutate( + randomness: Randomness, + apc: AdaptiveParameterControl, + mwc: MutationWeightControl, + selectionStrategy: SubsetGeneMutationSelectionStrategy, + enableAdaptiveGeneMutation: Boolean, + additionalGeneMutationInfo: AdditionalGeneMutationInfo? + ): Boolean { + return false + } + + /** + * Zero-width: never contributes characters to the generated string. + */ + override fun getValueAsPrintableString( + previousGenes: List, + mode: GeneUtils.EscapeMode?, + targetFormat: OutputFormat?, + extraCheck: Boolean + ): String = "" + + /** + * Returns the value currently sampled from [innerGene], or null if there is no + * inner gene to sample from. + */ + fun sampledInnerValue(): String? { + if (innerGene == null) { + return null + } + return innerGene.getValueAsPrintableString(targetFormat = null) + } + + override fun containsSameValueAs(other: Gene): Boolean { + if (other !is AssertionRxGene) { + return false + } + if (assertionType != other.assertionType) { + return false + } + return sampledInnerValue() == other.sampledInnerValue() + } + + override fun unsafeCopyValueFrom(other: Gene): Boolean { + if (other !is AssertionRxGene || assertionType != other.assertionType) { + return false + } + return if (innerGene != null && other.innerGene != null) { + innerGene.unsafeCopyValueFrom(other.innerGene) + } else { + innerGene == null && other.innerGene == null + } + } + + /** + * Always true: an assertion never renders characters ([getValueAsPrintableString] is + * always ""), so it can always collapse to zero width. + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = true + + /** + * Always 0: an assertion never absorbs candidate text into itself, it only supplies a + * candidate for its siblings to absorb, via [sampledInnerValue]. + * @see [RxAbsorbable.tryForce] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + return 0 + } + + /** + * No-op: already always zero-width, so there's nothing to force. + * @see [RxAbsorbable.forceZeroWidth] + */ + override fun forceZeroWidth() { + // already always zero-width, nothing to do + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/BackReferenceRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/BackReferenceRxGene.kt index 8a85458d28..63006020e2 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/BackReferenceRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/BackReferenceRxGene.kt @@ -81,4 +81,20 @@ class BackReferenceRxGene( // nothing to copy, as the value comes from the capture group return containsSameValueAs(other) } + + /** + * Returns false as we do not want backreferences to mutate a previous group. + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = false + + /** + * Always 0: a backreference's value is derived entirely from a previous [captureGroup], + * so unlike an ordinary leaf it can not be forced to absorb arbitrary candidate text. + * @see [RxAbsorbable.tryForce] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + return 0 + } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt index a127e16055..32081c17ac 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterClassEscapeRxGene.kt @@ -115,6 +115,13 @@ class CharacterClassEscapeRxGene( companion object{ private val log = LoggerFactory.getLogger(CharacterRangeRxGene::class.java) + private fun Char.swapCase(): Char = + if (this.isUpperCase()) { + this.lowercaseChar() + } else { + this.uppercaseChar() + } + private val digitSet = listOf(CharacterRange('0', '9')) private val asciiLetterSet = listOf(CharacterRange(FIRST_LOWER_CASE_CHAR, LAST_LOWER_CASE_CHAR), CharacterRange(FIRST_UPPER_CASE_CHAR, LAST_UPPER_CASE_CHAR)) @@ -352,4 +359,49 @@ class CharacterClassEscapeRxGene( return false } + /** + * 1 if [value]'s first character (or its case-swapped counterpart, when caseable) is + * within [multiCharRange], else 0. + * @see [RxAbsorbable.absorbableCount] + */ + override fun absorbableCount(value: String): Int { + if (value.isEmpty()) { + return 0 + } + val c = value[0] + if (multiCharRange.contains(c)) { + return 1 + } + if (flags.isCaseable(c) && multiCharRange.contains(c.swapCase())) { + return 1 + } + return 0 + } + + /** + * Always false: a character-class escape always renders exactly one character. + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = false + + /** + * Forces [value]'s first character onto this gene, swapping case if that's what + * matched; mirrors [absorbableCount]. + * @see [RxAbsorbable.tryForce] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + val n = absorbableCount(value) + require(n == 0 || n == 1) + if (n == 1) { + val c = value[0] + this.value = (if (multiCharRange.contains(c)) { + c + } else { + c.swapCase() + }).toString() + this.useUpperCase = c.isUpperCase() + } + return n + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterRangeRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterRangeRxGene.kt index 139bda468b..8a61ef7bd3 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterRangeRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/CharacterRangeRxGene.kt @@ -28,6 +28,13 @@ class CharacterRangeRxGene( companion object{ private val log = LoggerFactory.getLogger(CharacterRangeRxGene::class.java) + + private fun Char.swapCase(): Char = + if (this.isUpperCase()) { + this.lowercaseChar() + } else { + this.uppercaseChar() + } } // '\u0000' is a placeholder for the unsatisfiable case (empty MCR with no valid ranges). @@ -180,4 +187,50 @@ class CharacterRangeRxGene( return false } + + /** + * 1 if [value]'s first character (or its case-swapped counterpart, when caseable) is + * within [validRanges], else 0. + * @see [RxAbsorbable.absorbableCount] + */ + override fun absorbableCount(value: String): Int { + if (value.isEmpty()) { + return 0 + } + val c = value[0] + if (validRanges.contains(c)) { + return 1 + } + if (flags.isCaseable(c) && validRanges.contains(c.swapCase())) { + return 1 + } + return 0 + } + + /** + * Always false: a character range always renders exactly one character. + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = false + + /** + * Forces [value]'s first character onto this gene, swapping case if that's what + * matched; mirrors [absorbableCount]. + * @see [RxAbsorbable.tryForce] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + val n = absorbableCount(value) + require(n == 0 || n == 1) + if (n == 1) { + val c = value[0] + this.value = if (validRanges.contains(c)) { + c + } else { + c.swapCase() + } + this.useUpperCase = c.isUpperCase() + } + return n + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionListRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionListRxGene.kt index 43676bcfb4..5307762eb0 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionListRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionListRxGene.kt @@ -15,6 +15,7 @@ import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutation import org.slf4j.Logger import org.slf4j.LoggerFactory +private data class BranchRanking(val absorbableCount: Int, val branchIndex: Int) class DisjunctionListRxGene( val disjunctions: List @@ -206,4 +207,132 @@ class DisjunctionListRxGene( return true } + + /** + * Ranks all branches by how much of [value] they can absorb, without mutating + * anything. Shared by both directions: [absorb] is [DisjunctionRxGene.absorbableCount] + * for lookahead's ranking, [DisjunctionRxGene.absorbableSuffixCount] for lookbehind's. + */ + private fun rankBranches( + value: String, + absorb: (DisjunctionRxGene, String) -> Int + ): BranchRanking? { + if (value.isEmpty() || disjunctions.isEmpty()) { + return null + } + var bestCount = absorb(disjunctions[activeDisjunction], value) + var bestIndex = activeDisjunction + for (i in disjunctions.indices) { + if (i == activeDisjunction) { + continue + } + if (bestCount == value.length) { + break + } + val can = absorb(disjunctions[i], value) + if (can > bestCount) { + bestCount = can + bestIndex = i + } + } + return BranchRanking(bestCount, bestIndex) + } + + /** + * Ranks every branch by how much of [value] it could absorb, without mutating, and + * reports the best [RxAbsorbable.absorbableCount]. + * @see [RxAbsorbable.absorbableCount] + * @see [rankBranches] + */ + override fun absorbableCount(value: String): Int = + rankBranches(value){ disjunction, value -> disjunction.absorbableCount(value) }?.absorbableCount ?: 0 + + /** + * True if at least one branch can render "", as we can select that branch and force it. + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = disjunctions.any { it.canBeZeroWidth } + + /** + * Shared rank-then-force logic behind [tryForce]/[tryForceSuffix]: ranks all branches via + * [absorb], switches [activeDisjunction] to the winner if needed, then delegates to [force] + * on that branch. + */ + private fun forceBestBranch( + value: String, + absorb: (DisjunctionRxGene, String) -> Int, + force: (DisjunctionRxGene, String) -> Int + ): Int { + val (bestCount, bestIndex) = rankBranches(value, absorb) ?: BranchRanking(0, activeDisjunction) + + if (bestCount > 0) { + if (bestIndex != activeDisjunction) { + activeDisjunction = bestIndex + tryToActivateGene(disjunctions[bestIndex]) + } + return force(disjunctions[bestIndex], value) + } + return 0 + } + + /** + * Activates whichever branch can best absorb [value] (switching [activeDisjunction] if + * needed) and forces it there. + * @see [RxAbsorbable.tryForce] + * @see [rankBranches] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + return forceBestBranch(value, + absorb = { d, v -> d.absorbableCount(v) }, + force = { d, v -> d.tryForce(v) } + ) + } + + /** + * Suffix counterpart of [absorbableCount]: ranks every branch by how much of + * [value]'s trailing characters it could absorb, without mutating. + * @see [RxAbsorbable.absorbableSuffixCount] + */ + override fun absorbableSuffixCount(value: String): Int = + rankBranches(value) { d, v -> d.absorbableSuffixCount(v) }?.absorbableCount ?: 0 + + /** + * Suffix counterpart of [tryForce]: activates whichever branch can best absorb + * [value]'s trailing characters and forces it there, walking right-to-left. + * @see [RxAbsorbable.tryForceSuffix] + */ + override fun tryForceSuffix(value: String): Int { + require(value.isNotEmpty()) + return forceBestBranch(value, + absorb = { d, v -> d.absorbableSuffixCount(v) }, + force = { d, v -> d.tryForceSuffix(v) } + ) + } + + /** + * Forces the active branch to zero width if it can; otherwise switches to the first + * branch that can and forces that one instead. + * @see [RxAbsorbable.forceZeroWidth] + */ + override fun forceZeroWidth() { + require(canBeZeroWidth) + // try the active branch first to avoid an unnecessary switch + val order = listOf(activeDisjunction) + disjunctions.indices.filter { it != activeDisjunction } + val target = order.first { disjunctions[it].canBeZeroWidth } + disjunctions[target].forceZeroWidth() + if (target != activeDisjunction) { + activeDisjunction = target + tryToActivateGene(disjunctions[target]) + } + } + + /** + * Delegates assertion repair to whichever branch is currently active, as only that + * branch's rendered value is ever observed, so only it needs repairing. See + * [DisjunctionRxGene.attemptAssertionRepair] for the actual repair logic. + */ + fun attemptAssertionRepair(randomness: Randomness) { + disjunctions.getOrNull(activeDisjunction)?.attemptAssertionRepair(randomness) + } } \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt index 4582495b4e..567b90d0a8 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/DisjunctionRxGene.kt @@ -4,6 +4,7 @@ import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.output.OutputFormat import org.evomaster.core.search.gene.root.CompositeFixedGene import org.evomaster.core.search.gene.Gene +import org.evomaster.core.search.gene.utils.AssertionRepairWalk import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.impact.impactinfocollection.regex.DisjunctionRxGeneImpact import org.evomaster.core.search.service.AdaptiveParameterControl @@ -14,6 +15,11 @@ import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutation import org.slf4j.Logger import org.slf4j.LoggerFactory +/** + * How many times a single [DisjunctionRxGene] tries to fix one of its own direct-term + * assertions, see [DisjunctionRxGene.attemptAssertionRepair]. + */ +const val MAX_LOCAL_ASSERTION_ATTEMPTS = 20 class DisjunctionRxGene( name: String, @@ -184,4 +190,112 @@ class DisjunctionRxGene( return ok } + /** + * Delegates to a forward walk over [terms]. + * @see [RxAbsorbable.absorbableCount] + * @see [AssertionRepairWalk.absorbableCount] + */ + override fun absorbableCount(value: String): Int = + AssertionRepairWalk.absorbableCount(terms, value) + + /** + * Delegates to a backward walk over [terms]. Mirrors [absorbableCount], walking + * right-to-left since lookbehind's target sits before the assertion. + * @see [RxAbsorbable.absorbableSuffixCount] + */ + override fun absorbableSuffixCount(value: String): Int = + AssertionRepairWalk.absorbableSuffixCount(terms, value) + + /** + * True only if every term can independently render "", as this disjunction's own value is + * the concatenation of all of them. + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = + terms.all { (it as RxAbsorbable).canBeZeroWidth } + + /** + * Delegates to a forward walk over [terms], mirroring [absorbableCount]. + * @see [RxAbsorbable.tryForce] + * @see [AssertionRepairWalk.tryForce] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + return AssertionRepairWalk.tryForce(terms, value) + } + + /** + * Delegates to a backward walk over [terms], mirroring [tryForce] in the opposite + * direction. + * @see [RxAbsorbable.tryForceSuffix] + */ + override fun tryForceSuffix(value: String): Int { + require(value.isNotEmpty()) + return AssertionRepairWalk.tryForceSuffix(terms, value) + } + + /** + * Forces every term to zero width individually. + * @see [RxAbsorbable.forceZeroWidth] + */ + override fun forceZeroWidth() { + require(canBeZeroWidth) + terms.forEach { (it as RxAbsorbable).forceZeroWidth() } + } + + /** + * Attempts to repair this disjunction's own value so that each of its direct-term + * [AssertionRxGene]s is actually satisfied, by forcing the assertion's sampled inner + * value onto the genes on the appropriate side of it within [terms]: + * - Forward, onto [terms] after it, for [AssertionType.LOOKAHEAD] + * - Backward, onto [terms] before it, for [AssertionType.LOOKBEHIND]. + */ + fun attemptAssertionRepair(randomness: Randomness) { + if (terms.none { it is AssertionRxGene }) { + return + } + + for (idx in terms.indices) { + val assertion = terms[idx] as? AssertionRxGene ?: continue + val innerGene = assertion.innerGene ?: continue + + val target = if (assertion.assertionType == AssertionType.LOOKBEHIND) { + terms.subList(0, idx).filter { it !is AssertionRxGene } + } else { + terms.subList(idx + 1, terms.size).filter { it !is AssertionRxGene } + } + + // we may not be able to force genes as target is empty but if the lookaround can be zero-width it is fine. + if (target.isEmpty()) { + if (innerGene.canBeZeroWidth) { + innerGene.forceZeroWidth() + continue + } + return + } + + val (countFunction, forceFunction) = + if (assertion.assertionType == AssertionType.LOOKBEHIND) { + AssertionRepairWalk::absorbableSuffixCount to AssertionRepairWalk::tryForceSuffix + } else { + AssertionRepairWalk::absorbableCount to AssertionRepairWalk::tryForce + } + + var satisfied = false + for (attempt in 0 until MAX_LOCAL_ASSERTION_ATTEMPTS) { + assertion.randomize(randomness, false) + val candidate = assertion.sampledInnerValue() ?: break + if (candidate.isEmpty() || countFunction(target, candidate) == candidate.length) { + if (candidate.isNotEmpty()) { + forceFunction(target, candidate) + } + satisfied = true + break + } + } + if (!satisfied) { + return + } + } + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/PatternCharacterBlockGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/PatternCharacterBlockGene.kt index cb1fd3cd54..a2d78499b4 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/PatternCharacterBlockGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/PatternCharacterBlockGene.kt @@ -115,4 +115,108 @@ class PatternCharacterBlockGene( return containsSameValueAs(other) } + /** + * Maps a forward walk-position [index] to the real index into [value], depending on [reversed]. + */ + private fun realIndex(reversed: Boolean, value: String, index: Int): Int = + if (reversed) { + value.lastIndex - index + } else { + index + } + + /** + * Shared body behind [absorbableCount]/[absorbableSuffixCount], they differ only in which + * end of [stringBlock] and [value] the walk anchors to, via [reversed]. No partial matches, + * returns 0 or the shortest string's length (which was matched). + */ + private fun matchCount(value: String, reversed: Boolean): Int { + var i = 0 + while (i < value.length && i < stringBlock.length) { + val blockIdx = realIndex(reversed, stringBlock, i) + val valueIdx = realIndex(reversed, value, i) + val c = stringBlock[blockIdx] + val matches = if (flags.isCaseable(c)) { + value[valueIdx].equals(c, ignoreCase = true) + } else { + value[valueIdx] == c + } + if (!matches) { + return 0 + } + i++ + } + return i + } + + /** + * Shared body behind [tryForce]/[tryForceSuffix]: commits the case (upper/lower) of the + * first (or last if [reversed]) [matchedLength] characters of [stringBlock] to match + * [value]'s corresponding characters. + */ + private fun applyForce(value: String, matchedLength: Int, reversed: Boolean) { + for (i in 0 until matchedLength) { + val blockIdx = realIndex(reversed, stringBlock, i) + val valueIdx = realIndex(reversed, value, i) + if (flags.isCaseable(stringBlock[blockIdx])) { + caseChoices[blockIdx] = value[valueIdx].isUpperCase() + } + } + } + + /** + * How many of [stringBlock]'s leading characters match [value]'s leading characters, + * case-insensitively wherever [flags] allows it. 0 when a character does not match, + * one of the strings must be consumed completely. + * @see [RxAbsorbable.absorbableCount] + */ + override fun absorbableCount(value: String): Int = matchCount(value, reversed = false) + + /** + * True only when [stringBlock] is empty, as a non-empty literal can never render "". + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = stringBlock.isEmpty() + + /** + * Commits the matching leading characters' case to match [value]; mirrors + * [absorbableCount] exactly. + * @see [RxAbsorbable.tryForce] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + val n = matchCount(value, reversed = false) + require(n == stringBlock.length || n == value.length || n == 0) + applyForce(value, matchedLength = n, reversed = false) + return n + } + + /** + * No-op: only reachable when [stringBlock] is empty, so there's nothing to place. + * @see [RxAbsorbable.forceZeroWidth] + */ + override fun forceZeroWidth() { + require(canBeZeroWidth) + } + + /** + * Suffix-anchored mirror of [absorbableCount]: how many of [stringBlock]'s trailing characters + * match [value]'s trailing characters, case-insensitively wherever [flags] allows it. + * 0 when a character does not match, one of the strings must be consumed completely. + * @see [RxAbsorbable.absorbableSuffixCount] + */ + override fun absorbableSuffixCount(value: String): Int = matchCount(value, reversed = true) + + /** + * Suffix-anchored mirror of [tryForce]: Commits the matching trailing characters' case to + * match [value]; mirrors [absorbableSuffixCount] exactly. + * @see [RxAbsorbable.tryForceSuffix] + */ + override fun tryForceSuffix(value: String): Int { + require(value.isNotEmpty()) + val n = matchCount(value, reversed = true) + require(n == stringBlock.length || n == value.length || n == 0) + applyForce(value, matchedLength = n, reversed = true) + return n + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt index 1ed5edf059..ebb010daa2 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/QuantifierRxGene.kt @@ -1,9 +1,9 @@ package org.evomaster.core.search.gene.regex -import org.evomaster.core.logging.LoggingUtil import org.evomaster.core.output.OutputFormat import org.evomaster.core.search.gene.root.CompositeGene import org.evomaster.core.search.gene.Gene +import org.evomaster.core.search.gene.utils.AssertionRepairWalk import org.evomaster.core.search.gene.utils.GeneUtils import org.evomaster.core.search.service.AdaptiveParameterControl import org.evomaster.core.search.service.Randomness @@ -240,4 +240,61 @@ class QuantifierRxGene( true } } + + /** + * Delegates to a forward walk over [atoms], this gene has no absorption logic beyond + * what its repeated atoms can each individually take. + * @see [RxAbsorbable.absorbableCount] + * @see [AssertionRepairWalk.absorbableCount] + */ + override fun absorbableCount(value: String): Int = + AssertionRepairWalk.absorbableCount(atoms, value) + + /** + * Delegates to a backward walk over [atoms], mirroring [absorbableCount] in the + * opposite direction. + * @see [RxAbsorbable.absorbableSuffixCount] + */ + override fun absorbableSuffixCount(value: String): Int = + AssertionRepairWalk.absorbableSuffixCount(atoms, value) + + /** + * True if zero repetitions are allowed ([min] == 0), or if [template] can itself render "". + * @see [RxAbsorbable.canBeZeroWidth] + */ + override val canBeZeroWidth: Boolean = + min == 0 || (template as? RxAbsorbable)?.canBeZeroWidth == true + + /** + * Delegates to a forward walk over [atoms], mirroring [absorbableCount]. + * @see [RxAbsorbable.tryForce] + * @see [AssertionRepairWalk.tryForce] + */ + override fun tryForce(value: String): Int { + require(value.isNotEmpty()) + return AssertionRepairWalk.tryForce(atoms, value) + } + + /** + * Delegates to a backward walk over [atoms], mirroring [tryForce]. + * @see [RxAbsorbable.tryForceSuffix] + */ + override fun tryForceSuffix(value: String): Int { + require(value.isNotEmpty()) + return AssertionRepairWalk.tryForceSuffix(atoms, value) + } + + /** + * Collapses to zero repetitions if [min] == 0 (removing every atom), otherwise forces + * each existing atom to zero width individually. + * @see [RxAbsorbable.forceZeroWidth] + */ + override fun forceZeroWidth() { + require(canBeZeroWidth) + if (min == 0) { + killAllChildren() + } else { + atoms.forEach { (it as RxAbsorbable).forceZeroWidth() } + } + } } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RegexGene.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RegexGene.kt index 4787152f1c..f09b7b67b6 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RegexGene.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RegexGene.kt @@ -11,8 +11,15 @@ import org.evomaster.core.search.service.Randomness import org.evomaster.core.search.service.mutator.MutationWeightControl import org.evomaster.core.search.service.mutator.genemutation.AdditionalGeneMutationInfo import org.evomaster.core.search.service.mutator.genemutation.SubsetGeneMutationSelectionStrategy +import org.evomaster.core.utils.RegexFlags +import org.evomaster.core.utils.RegexWithExternalFlags import java.util.regex.Pattern +/** + * How many times we try to resample the whole [RegexGene] and attempt repairing before we give up. + */ +const val MAX_TREE_REPAIR_ATTEMPTS = 10 + /** * A gene representing a regular expression (regex). */ @@ -29,21 +36,59 @@ class RegexGene( * so, this is a reasonable workaround */ var fixedValue: String? = null, - var usingFixedValue: Boolean = false + var usingFixedValue: Boolean = false, + val externalRegexFlags: RegexFlags = RegexFlags(), + val hasAssertions: Boolean = false ) : CompositeFixedGene(name, disjunctions) { override fun copyContent(): Gene { - return RegexGene(name, disjunctions.copy() as DisjunctionListRxGene, sourceRegex, regexType, fixedValue, usingFixedValue) + return RegexGene(name, disjunctions.copy() as DisjunctionListRxGene, sourceRegex, regexType, fixedValue, usingFixedValue, externalRegexFlags, hasAssertions) } + companion object { + private val patternCache = java.util.concurrent.ConcurrentHashMap() + + private fun compiledPattern(sourceRegex: String, flags: RegexFlags): Pattern { + return patternCache.computeIfAbsent(RegexWithExternalFlags(sourceRegex, flags)) { + Pattern.compile(sourceRegex, flags.toJavaFlagBitmask()) + } + } + } + + private val pattern: Pattern? = + if (regexType == RegexType.JVM) { + compiledPattern(sourceRegex, externalRegexFlags) + } else { + null + } + override fun randomize(randomness: Randomness, tryToForceNewValue: Boolean) { usingFixedValue = if(fixedValue == null){ false } else { randomness.nextBoolean() } - disjunctions.randomize(randomness, tryToForceNewValue) + + if (regexType != RegexType.JVM) { + disjunctions.randomize(randomness, tryToForceNewValue) + return + } + + repeat(MAX_TREE_REPAIR_ATTEMPTS) { _ -> + disjunctions.randomize(randomness, tryToForceNewValue) + if (pattern!!.matcher(disjunctions.getValueAsPrintableString()).find()) { + return + } + if (hasAssertions) { + disjunctions.attemptAssertionRepair(randomness) + if (pattern.matcher(disjunctions.getValueAsPrintableString()).find()) { + return + } + } + } + + throw IllegalStateException("Could not repair regex value") } @Deprecated("Do not call directly outside this package. Call setFromStringValue") @@ -66,7 +111,7 @@ class RegexGene( if(regexType == RegexType.JVM){ val matcher = try{ - Pattern.compile(sourceRegex).matcher(fixedValue!!) + pattern!!.matcher(fixedValue!!) }catch(e: Exception){ return false } diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RxAbsorbable.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RxAbsorbable.kt new file mode 100644 index 0000000000..9056ff6581 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RxAbsorbable.kt @@ -0,0 +1,72 @@ +package org.evomaster.core.search.gene.regex + +/** + * Defines the interface for regex genes that can "absorb" part of a candidate + * string during assertion repair. + * + * When a sampled string fails to satisfy an assertion, the repair loop samples a + * candidate string from [AssertionRxGene.innerGene] and greedily walks the genes + * around the assertion in its parent [DisjunctionRxGene]'s terms list, + * asking each gene how much of the remaining candidate it can absorb and committing + * that amount before moving to the next gene. + */ +interface RxAbsorbable { + + /** + * Read-only: maximum number of leading characters of [value] this gene can be forced to + * produce, or already produces without needing to change. Default 0 = cannot help. + */ + fun absorbableCount(value: String): Int = 0 + + /** + * Places as many of [value]'s leading characters as possible and returns how many characters were actually placed. + * + * Precondition: [value] is not empty. + */ + fun tryForce(value: String): Int { + throw IllegalStateException( + "${this::class.simpleName} does not support forcing but tryForce was called" + ) + } + + /** + * Read-only: could [forceZeroWidth] succeed on this gene (i.e.: "" can match this expression)? + */ + val canBeZeroWidth: Boolean + + /** + * Forces this gene to render as "". Unlike [tryForce], there is nothing to plan or report. + * + * Precondition: [canBeZeroWidth] + */ + fun forceZeroWidth() { + throw IllegalStateException( + "${this::class.simpleName} cannot be zero-width but forceZeroWidth was called" + ) + } + + /** + * Suffix-anchored counterpart of [absorbableCount], used by lookbehind repair: how many of + * [value]'s trailing characters could this gene be forced to produce. + * + * Default delegates to [absorbableCount] fed reversed [value]. This default is + * used by genes that produce exactly one character, like [AnyCharacterRxGene]. + */ + fun absorbableSuffixCount(value: String): Int = + if (value.isEmpty()) 0 else absorbableCount(value.reversed()) + + /** + * Suffix-anchored counterpart of [tryForce]: places as many of [value]'s trailing characters + * as possible and returns how many were actually placed. + * + * Default delegates to [tryForce] fed reversed [value], in a similar way to [absorbableSuffixCount]. + * + * Precondition: [value] is not empty. + */ + fun tryForceSuffix(value: String): Int { + require(value.isNotEmpty()) + val n = absorbableSuffixCount(value) + if (n > 0) tryForce(value.reversed()) + return n + } +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RxTerm.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RxTerm.kt index d6ccdf3574..dd34fac2bd 100644 --- a/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RxTerm.kt +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/regex/RxTerm.kt @@ -4,7 +4,7 @@ import org.evomaster.core.search.StructuralElement import org.evomaster.core.search.gene.Gene -interface RxTerm { +interface RxTerm: RxAbsorbable { /** * Returns true if this gene can never produce a valid value, * for example an empty character class intersection like [a&&b]. diff --git a/core/src/main/kotlin/org/evomaster/core/search/gene/utils/AssertionRepairWalk.kt b/core/src/main/kotlin/org/evomaster/core/search/gene/utils/AssertionRepairWalk.kt new file mode 100644 index 0000000000..5a1dbb3322 --- /dev/null +++ b/core/src/main/kotlin/org/evomaster/core/search/gene/utils/AssertionRepairWalk.kt @@ -0,0 +1,101 @@ +package org.evomaster.core.search.gene.utils + +import org.evomaster.core.search.gene.Gene +import org.evomaster.core.search.gene.regex.RxAbsorbable + +/** + * Greedy-walk utilities for assertion repair. + * + * Given a candidate string sampled from [org.evomaster.core.search.gene.regex.AssertionRxGene.innerGene], attempts to place + * as much of it as possible into a sequence of genes by asking each gene (via [org.evomaster.core.search.gene.regex.RxAbsorbable]) + * how much it can take and committing that amount. + */ +object AssertionRepairWalk { + /** + * Shared algorithm behind all public functions below, these differ only in: + * - [absorb]: which [RxAbsorbable] operation to call per gene (a read-only count, or a mutating force) + * - [onZeroWidth]: what to do when [absorb] returns 0 and the gene [RxAbsorbable.canBeZeroWidth] + * - [reversed]: whether to walk [genes] right-to-left for lookbehind or left-to-right for lookahead + */ + private fun walk( + genes: List, + value: String, + absorb: (RxAbsorbable, String) -> Int, + onZeroWidth: (RxAbsorbable) -> Unit, + reversed: Boolean + ): Int { + if (value.isEmpty()) { + return 0 + } + var consumed = 0 + val walkTarget = if (reversed) { + genes.asReversed() + } else { + genes + } + for (gene in walkTarget) { + if (consumed >= value.length) { + break + } + val absorbable = gene as RxAbsorbable + val remaining = if (reversed) { + value.dropLast(consumed) + } else { + value.substring(consumed) + } + val amount = absorb(absorbable, remaining) + if (amount == 0) { + if (absorbable.canBeZeroWidth) { + onZeroWidth(absorbable) + continue + } + return 0 + } + consumed += amount + } + return consumed + } + + /** + * Maximum leading characters of [value] that can be absorbed across [genes] + * left-to-right, without mutating anything. + */ + fun absorbableCount(genes: List, value: String): Int = + walk(genes, value, reversed = false, + absorb = { gene, value -> gene.absorbableCount(value) }, + onZeroWidth = {} + ) + + /** + * Forces as much of [value] as possible into [genes] left-to-right, mutating each + * gene in place using each gene's [RxAbsorbable.tryForce]. Returns total characters placed. + */ + fun tryForce(genes: List, value: String): Int = + walk(genes, value, reversed = false, + absorb = { gene, value -> gene.tryForce(value) }, + onZeroWidth = { it.forceZeroWidth() } + ) + + /** + * Suffix-anchored counterpart of [absorbableCount], used by lookbehind repair: maximum + * trailing characters of [value] that can be absorbed across [genes] right-to-left + * (walking [genes] in reverse, since the gene closest to the assertion's position is the + * last one in [genes]), without mutating anything. + */ + fun absorbableSuffixCount(genes: List, value: String): Int = + walk(genes, value, reversed = true, + absorb = { gene, value -> gene.absorbableSuffixCount(value) }, + onZeroWidth = {} + ) + + /** + * Suffix-anchored counterpart of [tryForce], used by lookbehind repair: forces as much + * of [value] as possible into [genes] right-to-left (mirroring [tryForce]), mutating each + * gene in place using [RxAbsorbable.tryForceSuffix]. Returns total characters placed. + */ + fun tryForceSuffix(genes: List, value: String): Int = + walk(genes, value, reversed = true, + absorb = { gene, value -> gene.tryForceSuffix(value) }, + onZeroWidth = { it.forceZeroWidth() } + ) +} \ No newline at end of file diff --git a/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt b/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt index 77c8b5f680..477149a524 100644 --- a/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt +++ b/core/src/main/kotlin/org/evomaster/core/utils/MultiCharacterRange.kt @@ -204,6 +204,24 @@ class MultiCharacterRange internal constructor(val ranges: List) return intersect(this, other) } + /** + * Check if [c] is contained in this [MultiCharacterRange]. + */ + fun contains(c: Char): Boolean { + if (isEmpty) { + return false + } + // binary search as MultiCharacterRange's ranges are ordered and non-overlapping + val index = ranges.binarySearch { (start, end) -> + when { + c < start -> 1 // Target is before this range, search left + c > end -> -1 // Target is after this range, search right + else -> 0 // Target is inside this range, match + } + } + return index >= 0 + } + val isEmpty: Boolean get() = ranges.isEmpty() val isNotEmpty: Boolean get() = ranges.isNotEmpty() val size: Int get() = ranges.size diff --git a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt index 27fe06280b..19d7ba5216 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/GeneRegexJavaVisitorTest.kt @@ -453,4 +453,59 @@ class GeneRegexJavaVisitorTest : GeneRegexEcma262VisitorTest() { fun testUnicodeCharClassFlagImpliesUnicodeCase(){ checkCanSample("(?iU)Å", "å", 100) } + + @Test + fun testSimpleLookaheads() { + checkSameAsJava("(?=aaa5)aaa\\d") + checkSameAsJava("(?=.*\\d).{4,8}") + checkSameAsJava("(?=.*[A-Z])[a-zA-Z]{4,8}") + checkSameAsJava("(?=.*[a-z])[a-zA-Z]{4,8}") + checkSameAsJava("foo(?=.*\\d)[a-z\\d]{3,6}") + checkSameAsJava("(?=.*\\d)\\w{6,12}") + checkSameAsJava("(?=.*[^A-Za-z0-9])\\w{6,12}$") + checkSameAsJava("^(?=.*\\d)[a-zA-Z\\d]{8,16}$") + checkSameAsJava("(?=\\d)\\d{1,5}") + checkSameAsJava("(?=.*\\d)([a-z]+|\\d+){2,4}") + checkSameAsJava("(?=(!.*[a-z]+))\\1") + checkSameAsJava("(?=(\\d|\\s|d))(\\d|\\s|d)*") + checkSameAsJava("(?=a*)\\w*") + checkSameAsJava("(?=xbcde)x(bcdX|bc)de") + checkSameAsJava("^(?=(\\S+))(\\d+h)?(\\d+m)?(\\d+s)?$") + checkSameAsJava("(?=ababc)(ab|abc)+") + } + + @Test + fun testUnsatisfiableLookaheads() { + assertThrows { checkSameAsJava("(?=.*\\d)(?=.*[A-Z])[a-zA-Z]{4,8}") } + assertThrows { checkSameAsJava("(?=.*\\d)(?=.*[A-Z])") } + assertThrows { checkSameAsJava("(?=.*\\d)[a-z]+") } + assertThrows { checkSameAsJava("(?=bbbX)aaa[a-z]") } + assertThrows { checkSameAsJava("(?=abcde)a(bcef|de)de") } + assertThrows { checkSameAsJava("(?=[a&&b])a(bcef|de)de") } + checkSameAsJava("abc|(?=[a&&b])def") + } + + @Test + fun testSimpleLookbehinds() { + checkSameAsJava("foo(?<=oo)\\d+") + checkSameAsJava("\\d(?<=[13579])") + checkSameAsJava("a(?<=a)b") + checkSameAsJava("\\w*(?<=z)c") + checkSameAsJava("[a-z]+(?<=aa|bb)cc") + checkSameAsJava("a(?<=a)b") + checkSameAsJava("(abc|ab|a)(?<=abc)") + checkSameAsJava("(?a)\\k") + } + + @Test + fun testLookbehindRepairAcrossDirections() { + checkSameAsJava("\\w+(?<=X*)m(?=z)\\w") + checkSameAsJava("^(?<=X*)m(?=z)(a|z)") + } + + @Test + fun testUnsatisfiableLookbehinds() { + assertThrows { checkSameAsJava("(?<=X)a") } + assertThrows { checkSameAsJava("a(?<=[a&&b])a") } + } } \ No newline at end of file diff --git a/core/src/test/kotlin/org/evomaster/core/parser/RegexHandlerTest.kt b/core/src/test/kotlin/org/evomaster/core/parser/RegexHandlerTest.kt index 1382ba3741..82f7480509 100644 --- a/core/src/test/kotlin/org/evomaster/core/parser/RegexHandlerTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/parser/RegexHandlerTest.kt @@ -14,7 +14,6 @@ import java.util.regex.Pattern internal class RegexHandlerTest{ - @Disabled("Needs to hande lookahead in regex") @Test fun testLanguageTool(){ val s = "^((?iu)@.+)$" diff --git a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneNumberOfGenesTest.kt b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneNumberOfGenesTest.kt index c5c7e608ff..7c3a1823fc 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneNumberOfGenesTest.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneNumberOfGenesTest.kt @@ -13,7 +13,7 @@ class GeneNumberOfGenesTest : AbstractGeneTest() { This number should not change, unless you explicitly add/remove any gene. if so, update this number accordingly */ - assertEquals(94, geneClasses.size) + assertEquals(95, geneClasses.size) } } diff --git a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneSamplerForTests.kt b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneSamplerForTests.kt index 97fe808654..7c42a6b9d7 100644 --- a/core/src/test/kotlin/org/evomaster/core/search/gene/GeneSamplerForTests.kt +++ b/core/src/test/kotlin/org/evomaster/core/search/gene/GeneSamplerForTests.kt @@ -143,6 +143,7 @@ object GeneSamplerForTests { QuantifierRxGene::class -> sampleQuantifierRxGene(rand) as T RegexGene::class -> sampleRegexGene(rand) as T BackReferenceRxGene::class -> sampleBackReferenceRxGene(rand) as T + AssertionRxGene::class -> sampleAssertionRxGene(rand) as T ObjectWithAttributesGene::class -> sampleObjectGeneWithAttributes(rand) as T //SQL genes @@ -428,6 +429,12 @@ object GeneSamplerForTests { ) } + fun sampleAssertionRxGene(rand: Randomness): AssertionRxGene { + val innerGene = sampleDisjunctionListRxGene(rand) + innerGene.doInitialize(rand) + return AssertionRxGene(innerGene=innerGene, AssertionType.LOOKAHEAD) + } + fun sampleRegexGene(rand: Randomness): RegexGene { return RegexGene( name = "rand RegexGene", @@ -473,8 +480,8 @@ object GeneSamplerForTests { //let's avoid huge trees... .filter { (it.java != DisjunctionListRxGene::class.java && it.java != DisjunctionRxGene::class.java - && it.java != BackReferenceRxGene::class.java) // as this also contains a DisjunctionListRxGene within - || rand.nextBoolean() + && it.java != BackReferenceRxGene::class.java && it.java != AssertionRxGene::class.java) // as this also contains a DisjunctionListRxGene within + || rand.nextBoolean(0.2) // reduced chance for larger trees } val numberOfTerms = rand.nextInt(1, 3)