diff --git a/.claude/settings.local.json b/.claude/settings.local.json index c782ec05..687926fe 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -29,7 +29,11 @@ "Bash(powershell:*)", "Bash(rustc --version)", "Bash(git config:*)", - "Bash(../target/release/wfl.exe pattern.wfl)" + "Bash(../target/release/wfl.exe pattern.wfl)", + "Bash(git fetch --all --prune)", + "Bash(git merge --no-ff:*)", + "Bash(git add -A)", + "Bash(git commit -m \"wfl-ai: *\")" ], "deny": [] } diff --git a/Cargo.lock b/Cargo.lock index d7781047..a6ad7dd0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3029,7 +3029,7 @@ dependencies = [ [[package]] name = "wfl" -version = "25.8.18" +version = "25.8.23" dependencies = [ "chrono", "codespan-reporting", diff --git a/Docs/api/wfl-standard-library.md b/Docs/api/wfl-standard-library.md index b9ba4d46..32a48610 100644 --- a/Docs/api/wfl-standard-library.md +++ b/Docs/api/wfl-standard-library.md @@ -323,6 +323,104 @@ Replaces pattern matches in text. - **Returns**: Text - **Example**: `store cleaned as pattern.replace(pattern, text, "")` +### Pre-built Patterns + +WFL provides commonly used patterns in the standard library for validation and parsing tasks: + +#### `email_pattern` +Validates email addresses according to RFC 5322 standards. +- **Type**: Pattern +- **Usage**: `if user_email matches email_pattern:` +- **Example**: + ```wfl + store user_input as "user@example.com" + if user_input matches email_pattern: + display "Valid email address" + end if + ``` + +#### `url_pattern` +Matches HTTP and HTTPS URLs with optional ports and paths. +- **Type**: Pattern +- **Matches**: `http://`, `https://` URLs with domains, ports, and paths +- **Example**: + ```wfl + store link as "https://example.com:8080/path?query=value" + if link matches url_pattern: + display "Valid URL" + end if + ``` + +#### `phone_pattern` +Matches common phone number formats including US/Canada formats. +- **Type**: Pattern +- **Formats**: (XXX) XXX-XXXX, XXX-XXX-XXXX, XXX.XXX.XXXX +- **Example**: + ```wfl + store phone as "555-123-4567" + store result as find phone_pattern in phone + if result is not nothing: + display "Phone: " with result.match + end if + ``` + +#### `ipv4_pattern` +Matches IPv4 addresses (0.0.0.0 to 255.255.255.255). +- **Type**: Pattern +- **Example**: + ```wfl + store server_ip as "192.168.1.100" + if server_ip matches ipv4_pattern: + display "Valid IPv4 address" + end if + ``` + +#### `ipv6_pattern` +Matches IPv6 addresses in standard and compressed formats. +- **Type**: Pattern +- **Formats**: Full, compressed (::), and mixed IPv6 formats +- **Example**: + ```wfl + store ipv6_addr as "2001:0db8:85a3::8a2e:0370:7334" + if ipv6_addr matches ipv6_pattern: + display "Valid IPv6 address" + end if + ``` + +#### `date_pattern` +Matches common date formats (YYYY-MM-DD, MM/DD/YYYY, DD-MM-YYYY). +- **Type**: Pattern +- **Example**: + ```wfl + store date_input as "2025-08-10" + store result as find date_pattern in date_input + if result is not nothing: + display "Found date: " with result.match + end if + ``` + +#### `time_pattern` +Matches time formats (HH:MM, HH:MM:SS, 12/24 hour with AM/PM). +- **Type**: Pattern +- **Example**: + ```wfl + store time_input as "14:30:45" + if time_input matches time_pattern: + display "Valid time format" + end if + ``` + +#### `uuid_pattern` +Matches UUID/GUID formats (8-4-4-4-12 hexadecimal pattern). +- **Type**: Pattern +- **Example**: + ```wfl + store session_id as "550e8400-e29b-41d4-a716-446655440000" + if session_id matches uuid_pattern: + display "Valid UUID" + end if + ``` + ## Type System Integration All standard library functions are integrated with WFL's type checker: diff --git a/Docs/wfl-devin.md b/Docs/dev-notes/wfl-devin.md similarity index 100% rename from Docs/wfl-devin.md rename to Docs/dev-notes/wfl-devin.md diff --git a/Docs/wfl-gemini-research.md b/Docs/dev-notes/wfl-gemini-research.md similarity index 100% rename from Docs/wfl-gemini-research.md rename to Docs/dev-notes/wfl-gemini-research.md diff --git a/Docs/wfl-int2.md b/Docs/dev-notes/wfl-int2.md similarity index 100% rename from Docs/wfl-int2.md rename to Docs/dev-notes/wfl-int2.md diff --git a/Docs/wfl-library-recommendations.md b/Docs/dev-notes/wfl-library-recommendations.md similarity index 100% rename from Docs/wfl-library-recommendations.md rename to Docs/dev-notes/wfl-library-recommendations.md diff --git a/Docs/wfl-rust-loc-counter.md b/Docs/dev-notes/wfl-rust-loc-counter.md similarity index 100% rename from Docs/wfl-rust-loc-counter.md rename to Docs/dev-notes/wfl-rust-loc-counter.md diff --git a/Docs/wfl-rust-loc-report.md b/Docs/dev-notes/wfl-rust-loc-report.md similarity index 100% rename from Docs/wfl-rust-loc-report.md rename to Docs/dev-notes/wfl-rust-loc-report.md diff --git a/Docs/wfl-todo.md b/Docs/dev-notes/wfl-todo.md similarity index 100% rename from Docs/wfl-todo.md rename to Docs/dev-notes/wfl-todo.md diff --git a/docs/BUILDING.md b/Docs/guides/building.md similarity index 100% rename from docs/BUILDING.md rename to Docs/guides/building.md diff --git a/Docs/wfl-pattern-migration.md b/Docs/guides/pattern-migration-guide.md similarity index 98% rename from Docs/wfl-pattern-migration.md rename to Docs/guides/pattern-migration-guide.md index 77bc86df..259a1240 100644 --- a/Docs/wfl-pattern-migration.md +++ b/Docs/guides/pattern-migration-guide.md @@ -562,7 +562,7 @@ end pattern If you encounter migration issues: -1. **Check Documentation:** Review the [Pattern Guide](wfl-pattern-guide.md) and [Unicode Patterns](wfl-unicode-patterns.md) +1. **Check Documentation:** Review the [Pattern Reference](../language-reference/wfl-patterns.md) and [Standard Library Patterns](../api/wfl-standard-library.md) 2. **Test Incrementally:** Migrate patterns one at a time 3. **Use Debug Mode:** Run patterns with `--debug` flag to see execution traces 4. **Create Minimal Examples:** Isolate problematic patterns for testing diff --git a/Docs/wfl-deployment.md b/Docs/guides/wfl-deployment.md similarity index 100% rename from Docs/wfl-deployment.md rename to Docs/guides/wfl-deployment.md diff --git a/Docs/implementation_progress_2025-08-04.md b/Docs/implementation_progress_2025-08-04.md deleted file mode 100644 index 768c69d4..00000000 --- a/Docs/implementation_progress_2025-08-04.md +++ /dev/null @@ -1,9 +0,0 @@ -# Implementation Progress - 2025-08-04 - - -## MSI Build - 22:08:54 - -- Version: 2025.57 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-2025.57.msi` - diff --git a/Docs/implementation_progress_2025-08-05.md b/Docs/implementation_progress_2025-08-05.md deleted file mode 100644 index 8bbdacc6..00000000 --- a/Docs/implementation_progress_2025-08-05.md +++ /dev/null @@ -1,51 +0,0 @@ -# Implementation Progress - 2025-08-05 - - -## MSI Build - 00:05:18 - -- Version: 2025.57 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-2025.57.msi` - - -## MSI Build - 00:16:25 - -- Version: 25.3 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-25.3.msi` - - -## MSI Build - 00:19:22 - -- Version: 25.3 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-25.3.msi` - - -## MSI Build - 00:30:20 - -- Version: 25.3 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-25.3.msi` - - -## MSI Build - 09:46:14 - -- Version: 25.3 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-25.3.msi` - - -## MSI Build - 12:22:08 - -- Version: 25.3 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-25.3.msi` - - -## MSI Build - 12:23:22 - -- Version: 25.3 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-25.3.msi` - diff --git a/Docs/implementation_progress_2025-08-06.md b/Docs/implementation_progress_2025-08-06.md deleted file mode 100644 index 8bd93ef6..00000000 --- a/Docs/implementation_progress_2025-08-06.md +++ /dev/null @@ -1,9 +0,0 @@ -# Implementation Progress - 2025-08-06 - - -## MSI Build - 09:27:54 - -- Version: 25.3 -- Status: SUCCESS -- Output: `target/x86_64-pc-windows-msvc/release/wfl-25.3.msi` - diff --git a/Docs/language-reference/wfl-control-flow.md b/Docs/language-reference/wfl-control-flow.md index d28eda48..3948dfde 100644 --- a/Docs/language-reference/wfl-control-flow.md +++ b/Docs/language-reference/wfl-control-flow.md @@ -1,54 +1,581 @@ -# Control Flow Handling in the WFL Interpreter +# WFL Control Flow Reference -The WFL interpreter implements structured control flow handling for loops and functions using a dedicated `ControlFlow` enum. This allows for proper handling of `break`, `continue`, `exit`, and `return` statements within nested structures. +Control flow statements in WFL allow you to direct the execution path of your program using natural, English-like syntax. This guide covers conditional statements, loops, and flow control keywords that make your programs dynamic and responsive. -## Control Flow Enum +## Overview -The `ControlFlow` enum is defined in `src/interpreter/control_flow.rs` with the following variants: +WFL provides intuitive control flow constructs that read like natural English: +- **Conditionals**: Make decisions with `check if`, `otherwise if`, and `otherwise` +- **Loops**: Repeat actions with `count`, `for each`, `repeat while/until`, and more +- **Flow Control**: Direct execution with `break`, `continue`/`skip`, and `give back` (alias: `return`) -```rust -pub enum ControlFlow { - None, // Normal execution, no control flow change - Break, // Break out of the current loop - Continue, // Skip to the next iteration of the current loop - Exit, // Exit from the outer loop (used by WFL's `exit loop`) - Return(Value), // Return a value from an action/function -} +All control flow structures use clear start and end markers, making code blocks easy to identify and understand. + +### Control Flow Block Pairs + +Each control flow construct has a matching opener and closer pair: + +| Construct | Opener | Closer | +|-----------|--------|--------| +| Conditional | `check if` | `end check` | +| Count Loop | `count from` | `end count` | +| For-Each Loop | `for each` | `end for` | +| Repeat Loop | `repeat while`/`repeat until`/`repeat forever` | `end repeat` | +| Main Loop | `main loop` | `end loop` | + +## Conditional Statements + +### Basic If-Then-Else + +WFL uses `check if` blocks for conditional execution: + +```wfl +check if user is logged in: + display "Welcome back!" +end check +``` + +### With Else Clause + +Add an `otherwise` clause to handle the false case: + +```wfl +check if age is at least 18: + display "You can vote" +otherwise: + display "Too young to vote" +end check +``` + +### Multiple Conditions + +Chain conditions with `otherwise if`: + +```wfl +check if score is above 90: + display "Grade: A" +otherwise if score is above 80: + display "Grade: B" +otherwise if score is above 70: + display "Grade: C" +otherwise: + display "Grade: F" +end check +``` + +### Single-Line Conditionals + +For simple cases, use the inline form: + +```wfl +if temperature is below 0 then display "Freezing!" otherwise display "Not freezing" + +// Can also omit the otherwise part +if file exists then display "File found" +``` + +### Natural Language Conditions + +WFL supports readable comparison operators: + +```wfl +// Equality +check if name is "Alice": // equals +check if count is not 0: // not equals + +// Numeric comparisons +check if age is greater than 21: // > +check if price is less than 100: // < +check if score is at least 50: // >= +check if temp is at most 30: // <= + +// Boolean conditions +check if is active: // if true +check if is active is false: // if false (comparison form) +check if not (is active): // if false (grouped negation) + +// String operations +check if email contains "@": // substring check +check if text starts with "Hello": // prefix check +check if text ends with ".txt": // suffix check + +// Null checks +check if value is nothing: // null check +check if value is not nothing: // not null + +// List operations +check if list is empty: // empty check +check if item is in shopping list: // membership test +``` + +### Combining Conditions + +Use `and` and `or` to combine multiple conditions: + +```wfl +check if user is logged in and role is "admin": + display "Admin panel" +end check + +check if temperature is below 0 or temperature is above 40: + display "Extreme weather warning!" +end check + +// Complex combinations +check if (age is at least 18 and has license) or is supervised: + display "Can drive" +end check +``` + +## Loop Constructs + +### Counting Loops + +Iterate over numeric ranges with `count`: + +```wfl +// Basic counting +count from 1 to 5: + display "Iteration: " with count +end count +// Output: 1, 2, 3, 4, 5 + +// Count with custom step +count from 0 to 20 by 2: + display "Even: " with count +end count +// Output: 0, 2, 4, 6, 8, 10, 12, 14, 16, 18, 20 + +// Count backwards +count from 10 down to 1: + display "Countdown: " with count +end count +// Output: 10, 9, 8, 7, 6, 5, 4, 3, 2, 1 + +// Count backwards with step +count from 100 down to 0 by 10: + display count +end count +// Output: 100, 90, 80, 70, 60, 50, 40, 30, 20, 10, 0 +``` + +The loop variable `count` is automatically available within the loop body. + +**Scoping Note:** The loop variable is always named `count` and is lexically scoped to the loop body. In nested count loops, the inner `count` shadows the outer `count` variable. Currently, there is no syntax to rename or alias the loop variable - it's always `count`. + +### For-Each Loops + +Iterate over collections: + +```wfl +store fruits as ["apple", "banana", "orange"] + +// Basic iteration +for each fruit in fruits: + display "I like " with fruit +end for + +// Reverse iteration +for each fruit in fruits reversed: + display fruit +end for +// Output: orange, banana, apple +``` + +### Conditional Loops + +#### Repeat While + +Continue looping while a condition is true: + +```wfl +store attempts as 0 +repeat while attempts is less than 3: + store attempts as attempts plus 1 + display "Attempt " with attempts + check if login successful: + break + end check +end repeat +``` + +#### Repeat Until + +Continue looping until a condition becomes true: + +```wfl +store temperature as 20 +repeat until temperature is above 100: + store temperature as temperature plus 10 + display "Heating... Current temp: " with temperature +end repeat +display "Target temperature reached!" +``` + +### Infinite Loops + +#### Forever Loop + +A standard infinite loop that respects timeout settings: + +```wfl +store counter as 0 +repeat forever: + store counter as counter plus 1 + display "Processing item " with counter + + check if counter is 100: + break // Exit the loop + end check +end repeat +``` + +#### Main Loop + +Special infinite loop for long-running applications that **disables timeout**: + +```wfl +// Server application example +main loop: + wait for request + process request + + check if shutdown signal received: + display "Shutting down gracefully..." + break + end check +end loop ``` -## Control Flow Propagation +**Key Differences:** +- `repeat forever`: Subject to execution timeout (will error if runs too long) +- `main loop`: Bypasses timeout, designed for servers and continuous services -All statement execution functions return a tuple of `(Value, ControlFlow)` where: -- `Value` is the result value of the statement -- `ControlFlow` indicates any special control flow signal +## Loop Control Statements -Most statements return `ControlFlow::None`, indicating normal execution flow. Special statements return specific control flow signals: +### Break Statement -- `BreakStatement` returns `ControlFlow::Break` -- `ContinueStatement` returns `ControlFlow::Continue` -- `ExitStatement` returns `ControlFlow::Exit` -- `ReturnStatement` returns `ControlFlow::Return(value)` +Exits the current loop immediately: + +```wfl +for each item in items: + check if item is "stop": + break // Exit the loop + end check + display item +end for +display "Loop ended" +``` + +### Continue/Skip Statements + +Skips to the next iteration (both keywords work the same): + +```wfl +count from 1 to 10: + check if count is even: + continue // Skip even numbers + end check + display count +end count +// Output: 1, 3, 5, 7, 9 + +// 'skip' is a synonym for 'continue' +for each file in files: + check if file ends with ".tmp": + skip // Skip temporary files + end check + process file +end for +``` + +### Exit Statement + +Breaks out of nested loops: + +```wfl +count from 1 to 5: + display "Outer: " with count + count from 1 to 5: + display " Inner: " with count + check if count is 3: + exit loop // Exits BOTH loops + end check + end count +end count +display "Both loops exited" +``` + +**Note:** `exit` or `exit loop` breaks out of all enclosing loops, while `break` only exits the innermost loop. + +## Control Flow in Actions + +### Give Back (alias: return) + +The `give back` statement is used to return values from actions (functions). You can also use `return` as an alias for `give back`. Using `give back` outside of an action results in a compile-time error. + +```wfl +define action calculate sum: + needs: + numbers as list + gives back: + total as number + do: + store sum as 0 + for each num in numbers: + check if num is negative: + give back 0 // Early return + end check + store sum as sum plus num + end for + give back sum +end action + +// Using the action +store result as perform calculate sum with numbers as [1, 2, 3] +display "Sum: " with result +``` + +### Control Flow Interaction + +Control flow statements behave consistently in nested contexts: + +```wfl +define action process items: + needs: + items as list + gives back: + result as text + do: + for each item in items: + check if item is "abort": + give back "Aborted" // Returns from the entire action + end check + + check if item is "skip": + continue // Skips to next iteration + end check + + check if item is "done": + break // Exits the loop, continues with action + end check + + display "Processing: " with item + end for + + give back "Completed" +end action +``` + +## Common Patterns + +### Search Pattern + +Find an item in a collection: + +```wfl +store found as no +store target as "apple" + +for each item in shopping list: + check if item is target: + store found as yes + break // Stop searching once found + end check +end for + +check if found: + display "Found " with target +otherwise: + display target with " not in list" +end check +``` + +### Filter Pattern + +Process only certain items: + +```wfl +for each number in numbers: + // Skip negative numbers + check if number is less than 0: + continue + end check + + // Process positive numbers + display "Processing: " with number +end for +``` + +### Accumulator Pattern + +Build up a result: + +```wfl +store total as 0 +store count as 0 + +for each score in scores: + store total as total plus score + store count as count plus 1 +end for + +check if count is greater than 0: + store average as total divided by count + display "Average: " with average +end check +``` + +### Nested Loop with Early Exit + +Search in a 2D structure: + +```wfl +store found as no +store row_index as 0 + +for each row in grid: + store cell_index as 0 + for each cell in row: + check if cell is target: + display "Found at row " with row_index with ", column " with cell_index + store found as yes + exit loop // Exit both loops + end check + store cell_index as cell_index plus 1 + end for + store row_index as row_index plus 1 +end for + +check if not found: + display "Target not found in grid" +end check +``` + +### Event Processing Loop + +Continuous event processor: + +```wfl +main loop: + store event as get next event + + check if event is nothing: + wait 100 milliseconds + continue // Check again + end check + + check if event.type is "shutdown": + display "Shutdown requested" + break + end check + + // Process the event + perform handle event with event as event + + // Check for errors + check if error occurred: + log error + continue // Skip to next event + end check + + update statistics +end loop + +display "Event processor stopped" +``` + +## Best Practices + +### 1. Choose the Right Loop Type + +- Use `count` for numeric ranges +- Use `for each` for collections +- Use `repeat while/until` for condition-based loops +- Use `main loop` for servers and long-running services + +### 2. Clear Exit Conditions + +Always provide a way to exit infinite loops: + +```wfl +// Good: Clear exit condition +main loop: + check if should stop: + break + end check + // ... rest of loop +end loop + +// Bad: No exit condition +repeat forever: + // ... this could run forever +end repeat +``` + +### 3. Minimize Nesting + +Use early returns and continues to reduce nesting: + +```wfl +// Good: Early exit reduces nesting +for each item in items: + check if item is invalid: + continue // Skip invalid items + end check + + process item // Main logic not nested +end for + +// Less readable: Deeply nested +for each item in items: + check if item is valid: + process item // Main logic is nested + end check +end for +``` + +### 4. Descriptive Conditions + +Use WFL's natural language to make conditions self-documenting: + +```wfl +// Good: Self-documenting +check if user is logged in and subscription is active: + allow access +end check + +// Less clear: Using just variables +check if logged and active: + allow access +end check +``` -## Loop Handling +## Technical Notes -Each loop implementation (`ForeverLoop`, `CountLoop`, `RepeatWhileLoop`, `RepeatUntilLoop`, `ForEachLoop`) handles control flow signals as follows: +### Control Flow Implementation -- On `ControlFlow::Break` → Break out of the current loop only -- On `ControlFlow::Continue` → Skip to the next iteration of the current loop -- On `ControlFlow::Exit` → Propagate upward to allow breaking out of nested loops -- On `ControlFlow::Return` → Propagate upward to the function caller +Internally, WFL uses a control flow signaling system where: +- Each statement can return a control flow signal +- Signals propagate up through nested structures +- Loops and functions handle signals appropriately -## Function/Action Handling +This ensures consistent behavior across all contexts: +- `break` affects only the innermost loop +- `exit/exit loop` can break out of nested loops +- `return/give back` exits the entire function +- `continue/skip` jumps to the next iteration -When a function encounters a control flow signal: -- `Return` signals are consumed by the function, which returns the specified value -- Other control flow signals (`Break`, `Continue`, `Exit`) are propagated upward +### Performance Considerations -## Nested Structure Handling +- Loop control statements have minimal overhead +- `main loop` disables timeout checking for better performance +- Early exits (`break`, `return`) can improve performance by avoiding unnecessary iterations +- Condition evaluation short-circuits (in `and`/`or` expressions) -Control flow signals bubble up through nested structures: -- A `break` in a nested loop only breaks out of the innermost loop -- An `exit loop` breaks out of the outer loop, regardless of nesting depth -- A `return` in a nested loop or conditional exits the entire function +## See Also -This structured approach ensures that control flow statements work correctly in all contexts, including complex nested structures. +- [WFL Language Specification](wfl-spec.md) - Complete language reference +- [Main Loop Documentation](wfl-main-loop.md) - Details on long-running loops +- [Actions Documentation](wfl-actions.md) - Functions and return values +- [WFL by Example](../guides/wfl-by-example.md) - Practical examples \ No newline at end of file diff --git a/Docs/wfl-IO.md b/Docs/language-reference/wfl-io.md similarity index 100% rename from Docs/wfl-IO.md rename to Docs/language-reference/wfl-io.md diff --git a/Docs/language-reference/wfl-patterns.md b/Docs/language-reference/wfl-patterns.md new file mode 100644 index 00000000..03188186 --- /dev/null +++ b/Docs/language-reference/wfl-patterns.md @@ -0,0 +1,691 @@ +# Pattern Matching in WebFirst Language (WFL) + +WFL provides a powerful, natural-language pattern matching system that makes it easy to work with text patterns without the complexity of traditional regular expressions. + +## Table of Contents +- [Overview](#overview) +- [Basic Syntax](#basic-syntax) +- [Pattern Elements](#pattern-elements) +- [Built-in Functions](#built-in-functions) +- [Common Patterns](#common-patterns) +- [Advanced Features](#advanced-features) +- [Unicode Support](#unicode-support) +- [Performance & Optimization](#performance--optimization) +- [Migration from Regex](#migration-from-regex) +- [Design Philosophy](#design-philosophy) +- [Best Practices](#best-practices) + +## Overview + +The WFL pattern matching system uses declarative `create pattern` blocks that compile to an efficient bytecode virtual machine. This approach provides: + +- **Natural language syntax** - Use words like "one or more", "optional", "between 1 and 5" +- **Type safety** - Capture groups are typed as `Option` with flow-sensitive analysis +- **Performance protection** - Built-in guards against catastrophic backtracking +- **Clear error messages** - Helpful diagnostics for both syntax and runtime errors +- **Full Unicode support** - Handle text in any language or script +- **Bytecode compilation** - Optimized execution through VM + +## Basic Syntax + +### Creating Patterns + +```wfl +create pattern email_pattern: + one or more letter or digit or "." or "_" + "@" + one or more letter or digit or "." +end pattern +``` + +### Using Patterns + +```wfl +// Check if text matches a pattern +if "user@example.com" matches email_pattern: + display "Valid email!" +end if + +// Find matches and extract captures +store result as find email_pattern in "Contact: user@example.com" +if result is not nothing: + display "Found email: " with result["match"] +end if + +// Replace matches +store cleaned as replace email_pattern with "[EMAIL]" in "Send to user@example.com" + +// Split text on pattern +store parts as split "one,two,three" on pattern comma_pattern +``` + +## Pattern Elements + +### Character Classes + +Character classes define what types of characters to match at a specific position: + +| Pattern | What it matches | Example matches | +|---------|----------------|-----------------| +| `any letter` | Any uppercase or lowercase letter (A-Z, a-z) | "a", "B", "z", "Q" | +| `any digit` | Any numeric digit (0-9) | "0", "5", "9" | +| `any whitespace` | Spaces, tabs, newlines, carriage returns | " ", "\t", "\n" | +| `any punctuation` | Common punctuation marks | ".", "!", "?", ",", ";" | +| `any character` | Literally any single character | "a", "7", "@", " ", "€" | +| `letter` | Shorthand for `any letter` | "a", "Z" | +| `digit` | Shorthand for `any digit` | "0", "9" | +| `whitespace` | Shorthand for `any whitespace` | " ", "\t" | + +**Combined Classes:** +```wfl +any letter or digit // Alphanumeric +any letter or digit or "_" // Variable names +any character not in "xyz" // Exclusion +any character from "a" to "z" // Range +``` + +### Quantifiers + +Control how many times elements can repeat: + +| Pattern | What it means | Example | +|---------|---------------|---------| +| `zero or more` | Match 0 or unlimited times | `zero or more letters` | +| `one or more` | Match at least once | `one or more digits` | +| `optional` | Match 0 or 1 time | `optional whitespace` | +| `exactly N` | Match exactly N times | `exactly 3 digits` | +| `between N and M` | Match between N and M times | `between 2 and 4 letters` | +| `at least N` | Match N or more times | `at least 5 characters` | +| `at most N` | Match up to N times | `at most 10 digits` | + +### Sequences and Alternatives + +```wfl +// Sequence: patterns in order +"hello" then " " then "world" + +// Alternatives: any of these patterns +"yes" or "no" or "maybe" + +// Grouping with parentheses +("http" or "https") then "://" +``` + +### Anchors + +Match at specific positions in text: + +```wfl +at start of text // Beginning of entire text +at end of text // End of entire text +at start of line // Beginning of line +at end of line // End of line +word boundary // Word boundaries +``` + +### Captures and Backreferences + +Extract parts of matches and reference them later: + +```wfl +// Named capture groups +create pattern name_pattern: + capture {one or more letter} as "first_name" + whitespace + capture {one or more letter} as "last_name" +end pattern + +// Backreferences - match same content +create pattern duplicate_word: + capture {one or more letter} as "word" + whitespace + same as captured "word" +end pattern +``` + +### Lookarounds + +Zero-width assertions that check ahead or behind: + +```wfl +// Positive lookahead - must be followed by +digit followed by letter + +// Negative lookahead - must NOT be followed by +digit not followed by letter + +// Positive lookbehind - must be preceded by +digit preceded by "$" + +// Negative lookbehind - must NOT be preceded by +digit not preceded by "$" +``` + +## Built-in Functions + +### Pattern Operations + +| Function | Description | Example | +|----------|-------------|---------| +| `matches` | Check if text matches pattern | `if text matches email_pattern:` | +| `find` | Find first match with captures | `store result as find pattern in text` | +| `find_all` | Find all matches | `store all as find_all pattern in text` | +| `replace` | Replace matches | `store new as replace pattern with "X" in text` | +| `split` | Split by pattern | `store parts as split text on pattern delimiter` | + +#### find_all Function + +The `find_all` function finds all non-overlapping matches of a pattern in text and returns a list containing all matches with their capture groups. + +**Syntax:** +```wfl +store results as find_all pattern_name in text_value +``` + +**Return Type:** List of match objects, where each match contains: +- `match`: The full text that matched the pattern +- `captures`: Map of named capture groups to their values +- `position`: Starting position of the match in the original text + +**Example:** +```wfl +create pattern phone_number: + capture area: exactly 3 digit + "-" + capture exchange: exactly 3 digit + "-" + capture number: exactly 4 digit +end pattern + +store text as "Call 555-123-4567 or 555-987-6543 for help" +store all_phones as find_all phone_number in text + +count from each phone in all_phones: + display "Found phone: " with phone.match + display "Area code: " with phone.captures.area + display "Exchange: " with phone.captures.exchange + display "Number: " with phone.captures.number +end count + +// Output: +// Found phone: 555-123-4567 +// Area code: 555 +// Exchange: 123 +// Number: 4567 +// Found phone: 555-987-6543 +// Area code: 555 +// Exchange: 987 +// Number: 6543 +``` + +**Behavior:** +- Returns an empty list if no matches are found +- Matches do not overlap - after finding a match, search continues after the end of that match +- Capture groups that don't participate in a match contain `nothing` +- Matches are returned in the order they appear in the text + +### Standard Library Patterns + +WFL includes pre-built patterns for common use cases: + +```wfl +// Available in stdlib +email_pattern // Email validation +url_pattern // URL parsing +phone_pattern // Phone numbers +ipv4_pattern // IPv4 addresses +ipv6_pattern // IPv6 addresses +date_pattern // Date formats +time_pattern // Time formats +uuid_pattern // UUID format +``` + +## Common Patterns + +### Email Validation + +```wfl +create pattern email: + capture { + one or more letter or digit or "." or "_" or "%" or "+" or "-" + } as "username" + "@" + capture { + one or more letter or digit or "." or "-" + } as "domain" + "." + capture { + between 2 and 10 letter + } as "tld" +end pattern +``` + +### Phone Numbers + +```wfl +// US phone with flexible formatting +create pattern us_phone: + optional "+" or "1" then optional " " + optional "(" + capture {exactly 3 digit} as "area_code" + optional ")" + optional " " or "-" + capture {exactly 3 digit} as "exchange" + optional " " or "-" + capture {exactly 4 digit} as "line" +end pattern +``` + +### URL Parsing + +```wfl +create pattern url: + // Protocol + capture {optional ("http" or "https" or "ftp") then "://"} as "protocol" + + // Domain + capture { + one or more letter or digit or "-" + zero or more ("." then one or more letter or digit or "-") + } as "domain" + + // Port + optional (":" then capture {one or more digit} as "port") + + // Path + capture {optional ("/" then zero or more any character not in "?#")} as "path" + + // Query + optional ("?" then capture {zero or more any character not in "#"} as "query") + + // Fragment + optional ("#" then capture {zero or more any character} as "fragment") +end pattern +``` + +### Date Patterns + +```wfl +// ISO 8601 date +create pattern iso_date: + capture {exactly 4 digit} as "year" + "-" + capture {exactly 2 digit} as "month" + "-" + capture {exactly 2 digit} as "day" + optional ( + "T" + capture {exactly 2 digit} as "hour" + ":" + capture {exactly 2 digit} as "minute" + optional (":" then capture {exactly 2 digit} as "second") + optional ("Z" or ("+" or "-" then exactly 2 digit then ":" then exactly 2 digit)) + ) +end pattern +``` + +### Log Parsing + +```wfl +create pattern log_entry: + at start of line + capture {exactly 4 digit "-" exactly 2 digit "-" exactly 2 digit} as "date" + whitespace + capture {exactly 2 digit ":" exactly 2 digit ":" exactly 2 digit} as "time" + whitespace + "[" capture {one or more letter} as "level" "]" + whitespace + capture {one or more any character} as "message" +end pattern +``` + +## Advanced Features + +### Capture Groups and Extraction + +```wfl +create pattern name_pattern: + capture {one or more letter} as "first" + one or more whitespace + capture {one or more letter} as "last" +end pattern + +// Extract captured values +store result as find name_pattern in "John Doe" +if result is not nothing: + display "First: " with result["first"] + display "Last: " with result["last"] +end if +``` + +### Backreferences for Repeated Content + +```wfl +// Match HTML/XML tags +create pattern html_tag: + "<" + capture {one or more letter} as "tag" + ">" + zero or more any character + "" +end pattern + +// Validate balanced quotes +create pattern quoted_string: + capture {any of "\"" or "'"} as "quote" + zero or more any character not in captured "quote" + same as captured "quote" +end pattern +``` + +### Lookaround Assertions + +```wfl +// Password validation with lookarounds +create pattern strong_password: + // Must contain lowercase (positive lookahead) + followed by (zero or more any character then any lowercase) + + // Must contain uppercase (positive lookahead) + followed by (zero or more any character then any uppercase) + + // Must contain digit (positive lookahead) + followed by (zero or more any character then any digit) + + // Must contain special char (positive lookahead) + followed by (zero or more any character then any of "!@#$%^&*") + + // At least 8 characters + at least 8 any character +end pattern +``` + +### Pattern Composition + +```wfl +// Build complex patterns from simpler ones +create pattern word: + one or more letter +end pattern + +create pattern sentence: + word + zero or more (whitespace then word) + any of ".!?" +end pattern + +// Reuse patterns +create pattern paragraph: + sentence + zero or more (whitespace then sentence) +end pattern +``` + +## Unicode Support + +WFL provides comprehensive Unicode support for international text processing. + +### Unicode Categories + +```wfl +// Match any Unicode letter +create pattern unicode_letters: + unicode category "Letter" +end pattern + +// Match specific categories +create pattern uppercase: + unicode category "Uppercase_Letter" +end pattern + +create pattern currency: + unicode category "Currency_Symbol" // $, €, ¥, £, etc. +end pattern + +create pattern emoji: + unicode category "Other_Symbol" +end pattern +``` + +### Supported Unicode Categories + +**Letters:** +- `Letter` (L) - All letters +- `Uppercase_Letter` (Lu) - Uppercase letters +- `Lowercase_Letter` (Ll) - Lowercase letters +- `Titlecase_Letter` (Lt) - Titlecase letters +- `Modifier_Letter` (Lm) - Modifier letters +- `Other_Letter` (Lo) - Other letters (Chinese, Japanese, etc.) + +**Numbers:** +- `Number` (N) - All numbers +- `Decimal_Number` (Nd) - Decimal digits (0-9, ٠-٩, etc.) +- `Letter_Number` (Nl) - Letter-like numbers (Ⅰ, Ⅱ, etc.) +- `Other_Number` (No) - Other numbers (½, ¼, etc.) + +**Symbols:** +- `Symbol` (S) - All symbols +- `Math_Symbol` (Sm) - Math symbols (+, =, etc.) +- `Currency_Symbol` (Sc) - Currency symbols ($, €, ¥, etc.) +- `Modifier_Symbol` (Sk) - Modifier symbols +- `Other_Symbol` (So) - Other symbols (including emoji) + +**Punctuation:** +- `Punctuation` (P) - All punctuation +- `Connector_Punctuation` (Pc) - Connectors (_, ‿, etc.) +- `Dash_Punctuation` (Pd) - Dashes (-, –, —, etc.) +- `Open_Punctuation` (Ps) - Opening punctuation ((, [, {, etc.) +- `Close_Punctuation` (Pe) - Closing punctuation (), ], }, etc.) + +### Unicode Scripts + +```wfl +// Match specific writing systems +create pattern chinese_text: + unicode script "Han" +end pattern + +create pattern arabic_text: + unicode script "Arabic" +end pattern + +create pattern mixed_japanese: + unicode script "Hiragana" or "Katakana" or "Han" +end pattern +``` + +### International Examples + +```wfl +// Japanese email pattern +create pattern japanese_email: + one or more (unicode script "Hiragana" or "Katakana" or "Han" or letter or digit) + "@" + one or more (letter or digit or "." or "-") + "." + between 2 and 10 letter +end pattern + +// Arabic phone number +create pattern arabic_phone: + optional "+" + // Allow Arabic-Indic digits (٠١٢٣٤٥٦٧٨٩) or ASCII digits + exactly 3 (any of "٠١٢٣٤٥٦٧٨٩" or digit) + "-" + exactly 7 (any of "٠١٢٣٤٥٦٧٨٩" or digit) +end pattern +``` + +## Performance & Optimization + +### Pattern Compilation and Caching + +The WFL pattern engine automatically caches compiled patterns: + +```wfl +// Patterns are compiled once and cached +for each email in email_list: + if email matches email_pattern: // Uses cached bytecode + add email to valid_emails + end if +end for each +``` + +### Optimization Guidelines + +1. **Use specific patterns** rather than overly general ones +2. **Anchor patterns** when possible to reduce search space +3. **Use character classes** instead of long alternations +4. **Avoid nested quantifiers** that can cause backtracking +5. **Test with problematic inputs** to ensure performance + +### Performance Protections + +The WFL pattern engine includes: +- **Step counting** - Limits total matching operations +- **Recursion depth limits** - Prevents stack overflow +- **Backtracking guards** - Detects and prevents catastrophic backtracking +- **Memory pools** - Efficient memory management for captures + +## Migration from Regex + +### Conversion Guide + +| PCRE Regex | WFL Pattern | +|------------|-------------| +| `\d+` | `one or more digit` | +| `\w*` | `zero or more letter or digit or "_"` | +| `[a-zA-Z]+` | `one or more letter` | +| `\s+` | `one or more whitespace` | +| `(pattern)` | `capture {pattern} as "name"` | +| `pattern?` | `optional pattern` | +| `pattern{2,5}` | `between 2 and 5 pattern` | +| `pattern1\|pattern2` | `pattern1 or pattern2` | +| `^pattern$` | `at start of text then pattern then at end of text` | +| `(?=\d)` | `followed by digit` | +| `(?<=\$)` | `preceded by "$"` | +| `(.+)\1` | `capture {one or more any character} as "x" then same as captured "x"` | + +### Migration Strategy + +1. **Start simple** - Convert basic character classes first +2. **Test incrementally** - Verify each pattern works correctly +3. **Use composition** - Build complex patterns from simple ones +4. **Leverage tools** - Use conversion utilities where available + +## Design Philosophy + +WFL's pattern system represents a fundamental reimagining of text pattern matching. Traditional regular expressions, while powerful, suffer from a terse, symbol-heavy syntax that makes them notoriously difficult to read and maintain. + +### Why Natural Language Patterns? + +1. **Readability Over Brevity**: While regex prioritizes compact notation, WFL patterns prioritize clarity. Compare `^\d{3}-[A-Za-z]{2}$` with "at start of text then exactly 3 digit then '-' then exactly 2 letter then at end of text" - the latter is instantly understandable. + +2. **Self-Documenting Code**: WFL patterns serve as their own documentation. The pattern itself explains its purpose in plain English. + +3. **Lower Barrier to Entry**: By using familiar words instead of cryptic symbols, WFL makes pattern matching accessible to beginners and non-programmers. + +4. **Fewer Errors**: Natural language patterns eliminate common regex pitfalls like escaping issues, greedy vs. lazy quantifiers, and backreference confusion. + +### Historical Context + +WFL's pattern system draws inspiration from: +- **SNOBOL and Icon**: Languages that treated patterns as first-class objects +- **Raku (Perl 6)**: Introduced rules and grammars with more readable syntax +- **Parser Combinators**: Functional programming's composable parsers +- **Cucumber Expressions**: BDD tools using placeholders like `{int}` and `{string}` + +### Implementation Architecture + +WFL patterns compile to an efficient bytecode VM: + +``` +Pattern Source → Lexer → Parser → AST → Compiler → Bytecode → VM → Results +``` + +The VM uses: +- NFA simulation with epsilon transitions +- Backtracking with safety limits +- Parallel thread execution for alternatives +- Efficient capture group tracking + +## Best Practices + +### Pattern Design +1. **Use descriptive names** - `email_pattern` not `pattern1` +2. **Break complex patterns into parts** - Compose smaller patterns +3. **Test edge cases** - Empty strings, malformed input, boundaries +4. **Document intent** - Add comments explaining what patterns match + +### Error Handling +```wfl +store result as find pattern in text +if result is not nothing: + // Always check captures exist before using + if result["capture_name"] is not nothing: + display result["capture_name"] + end if +end if +``` + +### Performance +1. **Compile once, use many** - Let the engine cache patterns +2. **Profile hot patterns** - Measure and optimize frequently used patterns +3. **Use standard library** - Leverage pre-built, optimized patterns +4. **Avoid catastrophic backtracking** - Test with adversarial inputs + +### Maintainability +1. **Version control patterns** - Track changes like code +2. **Use pattern composition** - Build from reusable components +3. **Add unit tests** - Test patterns with known inputs/outputs +4. **Keep patterns simple** - Favor clarity over cleverness + +## Troubleshooting + +### Common Issues + +**Pattern doesn't match expected input:** +- Check for missing `optional` quantifiers +- Verify character classes match your data +- Test with simpler patterns first + +**Captures are empty:** +- Ensure capture groups actually match content +- Check quantifiers allow expected matches +- Verify capture names are used consistently + +**Performance issues:** +- Simplify complex alternations +- Reduce nested optional groups +- Use more specific patterns + +**Type errors with captures:** +- Always check `is not nothing` before using +- Remember captures are `Option` +- Use flow-sensitive analysis to narrow types + +### Error Messages + +WFL provides clear, actionable error messages: + +```wfl +// Syntax error +create pattern bad_range: + between 5 and 2 digit // Error: Invalid range +end pattern +// Error: PATTERN-SYNTAX-INVALID-RANGE +// Quantifier range must have min <= max + +// Runtime protection +// Error: PATTERN-RUNTIME-DEPTH +// Pattern matching stopped to prevent infinite loops +``` + +## See Also + +- [WFL Language Specification](wfl-spec.md) +- [Pattern Module API](../api/pattern-module.md) +- [Pattern Migration Guide](../guides/pattern-migration-guide.md) +- [Standard Library Reference](../api/wfl-standard-library.md) \ No newline at end of file diff --git a/Docs/language-reference/wfl-spec.md b/Docs/language-reference/wfl-spec.md index 35dc10c2..df164433 100644 --- a/Docs/language-reference/wfl-spec.md +++ b/Docs/language-reference/wfl-spec.md @@ -830,7 +830,92 @@ WFL abstracts away manual memory management, providing automatic memory handling - **Memory Model for Concurrency:** Since WFL supports async and possibly parallel tasks, one might wonder about memory consistency across threads. If WFL is single-threaded (like JavaScript event loop style), there’s no race condition issue. If it allows multi-threading (not explicitly indicated, likely not at the language level beyond async tasks that run concurrently but maybe still on one thread or using worker threads), the memory model would need to ensure thread-safe GC and possibly that shared data is synchronized. However, given the focus is on simplicity, WFL might abstract concurrency as asynchronous tasks in a single thread or cooperative multi-tasking. So we likely don’t have to specify a low-level memory model for threads (no atomic or volatile in WFL at the language level). -In conclusion, WFL uses a **managed memory approach** – either garbage collection or similar automatic reclamation – to handle memory safely. Developers can allocate freely (create lists, records, etc.) and trust the system to clean up. The absence of manual memory chores aligns with WFL’s goal of letting programmers focus on logic and not on pitfalls of memory management. +In conclusion, WFL uses a **managed memory approach** – either garbage collection or similar automatic reclamation – to handle memory safely. Developers can allocate freely (create lists, records, etc.) and trust the system to clean up. The absence of manual memory chores aligns with WFL's goal of letting programmers focus on logic and not on pitfalls of memory management. + +### Pattern Matching System + +WFL provides a comprehensive pattern matching system that uses natural language constructs to define text patterns, validate input, and extract data. The pattern matching system is formally specified with the following characteristics: + +#### Pattern Creation and Types + +Pattern definitions use declarative `create pattern` blocks that compile to efficient bytecode: + +```ebnf +PatternDecl ::= "create pattern" ":" "end pattern" +PatternBody ::= + +PatternExpression ::= | | | | +``` + +#### Capture Groups and Typing + +**Capture Group Semantics:** All capture groups in WFL patterns yield `Option` types, which represent either a captured text value or `nothing` if the group did not participate in the match: + +```wfl +create pattern phone_pattern: + capture area_code: exactly 3 digit + "-" + capture exchange: exactly 3 digit + "-" + capture number: exactly 4 digit +end pattern + +// Usage with flow-sensitive typing +store result as find phone_pattern in user_input +if result is not nothing: + // Within this block, result is known to contain captures + store area as result.area_code // Type: Option + if area is not nothing: + // Within this nested block, area is refined to Text + display "Area code: " with area + end if +end if +``` + +**Flow-Sensitive Type Analysis:** The type checker performs flow-sensitive analysis on pattern match results. Within conditional blocks that test for successful matches, captured values are automatically refined from `Option` to `Text` when null checks are performed. + +#### Unicode Property Support + +**Unicode Categories:** WFL supports matching by Unicode categories using the `unicode category` construct: + +```wfl +create pattern international_digits: + one or more unicode category "Decimal_Number" +end pattern +``` + +**Unicode Scripts:** Script-based matching uses the `unicode script` construct: + +```wfl +create pattern arabic_text: + one or more unicode script "Arabic" +end pattern +``` + +**Property Semantics:** Unicode property matching follows the Unicode Standard specifications. Categories include Letter, Number, Symbol, Punctuation, etc. Scripts include Latin, Arabic, Han, Cyrillic, etc. Invalid property names result in compile-time errors with suggested corrections. + +#### Performance Protection Systems + +**Backtracking Guards:** WFL's pattern matching engine includes built-in protection against catastrophic backtracking: + +1. **Step Counting:** Pattern matching operations are limited to a maximum number of backtracking steps (default: 1,000,000 steps). When exceeded, the pattern fails gracefully with a performance warning. + +2. **Recursion Depth Limits:** Nested pattern constructs (groups, alternations, quantifiers) are limited to a maximum depth (default: 100 levels) to prevent stack overflow. + +3. **Time Limits:** Pattern matching operations have configurable time limits (default: 5 seconds) to prevent indefinite execution. + +**Performance Configuration:** Applications can configure these limits using pattern options: + +```wfl +create pattern complex_pattern with options: + max_steps: 500000 + max_depth: 50 + timeout: 10 seconds + // pattern definition follows + ... +end pattern +``` + +**Error Handling:** When performance limits are exceeded, WFL returns a specific error type (`PatternPerformanceError`) that applications can handle gracefully without crashing. ## Conclusion The WebFirst Language brings together the above syntax and semantic rules to create a programming experience that is both beginner-friendly and powerful. Its formal grammar is defined to enforce consistency (so that tools can parse and compile it), but every rule in the grammar corresponds to a readable English-like construct. From **variables** (“Let X be Y” style declarations) to **control flow** (if/else and loops that read like instructions), **functions** (actions defined and called in descriptive ways), and **error handling** (“try ... when ...” blocks that narrate failure cases), WFL stays true to its guiding philosophy of **natural-language alignment, minimal symbols, clarity, and safety**. diff --git a/Docs/language-reference/wfl-variables.md b/Docs/language-reference/wfl-variables.md index f8f73636..bd238628 100644 --- a/Docs/language-reference/wfl-variables.md +++ b/Docs/language-reference/wfl-variables.md @@ -140,6 +140,41 @@ As shown: - `remove "eggs" from shopping` takes "eggs" out of the list. - `clear shopping list` empties the list completely. +#### Accessing List Items + +You can access individual items from a list in two ways: + +**Method 1: Using index notation (direct access)** +```wfl +create list shopping: + add "milk" + add "bread" + add "eggs" +end list + +// Access items directly using index (0-based) +store first_item as shopping 0 // "milk" +store second_item as shopping 1 // "bread" +store third_item as shopping 2 // "eggs" + +// You can use this in expressions too +display "First item: " + shopping 0 +store combined as shopping 0 + " and " + shopping 1 +``` + +**Method 2: Using the "item at" syntax** +```wfl +// Alternative syntax for accessing list items +store first as item at 0 from shopping +store second as item at 1 from shopping + +display "First item: " + first +``` + +Both methods work the same way. The index notation (`shopping 0`) is more concise and works well in expressions, while the "item at" syntax reads more like natural English. + +**Important:** List indices start at 0, so the first item is at index 0, the second at index 1, and so on. + ### Sets (Unique Collections) A set is for when you need to ensure all items are unique. Creating a set is similar to a list: diff --git a/docs/memory_profiling.md b/Docs/technical/memory-profiling.md similarity index 100% rename from docs/memory_profiling.md rename to Docs/technical/memory-profiling.md diff --git a/Docs/wfl-args.md b/Docs/technical/wfl-args.md similarity index 100% rename from Docs/wfl-args.md rename to Docs/technical/wfl-args.md diff --git a/Docs/wfl-lint.md b/Docs/technical/wfl-lint.md similarity index 100% rename from Docs/wfl-lint.md rename to Docs/technical/wfl-lint.md diff --git a/Docs/wfl-logging.md b/Docs/technical/wfl-logging.md similarity index 100% rename from Docs/wfl-logging.md rename to Docs/technical/wfl-logging.md diff --git a/Docs/wfl-oop-design.md b/Docs/technical/wfl-oop-design.md similarity index 100% rename from Docs/wfl-oop-design.md rename to Docs/technical/wfl-oop-design.md diff --git a/Docs/technical/wfl-staticTypeChecker.md b/Docs/technical/wfl-static-type-checker.md similarity index 100% rename from Docs/technical/wfl-staticTypeChecker.md rename to Docs/technical/wfl-static-type-checker.md diff --git a/Docs/wfl-step.md b/Docs/technical/wfl-step.md similarity index 100% rename from Docs/wfl-step.md rename to Docs/technical/wfl-step.md diff --git a/Docs/wfl-version.md b/Docs/technical/wfl-version.md similarity index 100% rename from Docs/wfl-version.md rename to Docs/technical/wfl-version.md diff --git a/Docs/wfl-documentation-index.md b/Docs/wfl-documentation-index.md index 583a60d6..824c63c1 100644 --- a/Docs/wfl-documentation-index.md +++ b/Docs/wfl-documentation-index.md @@ -1,6 +1,6 @@ # WFL Documentation Index -Welcome to the WebFirst Language documentation! This index provides a comprehensive guide to all available documentation. +Welcome to the WebFirst Language documentation! This index provides a comprehensive guide to all available documentation, organized for easy navigation according to the natural-language principles outlined in our [Foundation document](guides/wfl-foundation.md). ## 📚 Language Reference @@ -10,62 +10,78 @@ Core language documentation for learning and using WFL: - **[Variables Guide](language-reference/wfl-variables.md)** - Creating and using variables in WFL - **[Control Flow](language-reference/wfl-control-flow.md)** - Conditionals, loops, and program flow - **[Actions (Functions)](language-reference/wfl-actions.md)** - Defining and using actions +- **[Pattern Matching](language-reference/wfl-patterns.md)** - Comprehensive pattern matching with natural language syntax - **[Async Programming](language-reference/wfl-async.md)** - Asynchronous operations and concurrency - **[Container System](language-reference/wfl-containers.md)** - Object-oriented programming in WFL - **[Error Handling](language-reference/wfl-errors.md)** - Understanding and handling errors +- **[I/O Operations](language-reference/wfl-io.md)** - File and network input/output +- **[Main Loop](language-reference/wfl-main-loop.md)** - Event-driven programming -## 🔧 Technical Documentation +## 📖 Guides and Tutorials -Internal technical documentation for contributors and advanced users: +Best practices and learning resources: -- **[Lexer Implementation](technical/wfl-lexer.md)** - Tokenization and lexical analysis -- **[Lexer Fix Details](technical/lexer_fix_1.md)** - Documentation of lexer improvements -- **[Interpreter Design](technical/wfl-interpreter.md)** - AST execution and runtime -- **[Type Checker](technical/wfl-staticTypeChecker.md)** - Static type analysis system +- **[WFL Foundation](guides/wfl-foundation.md)** - Core principles and design philosophy +- **[Getting Started](guides/wfl-getting-started.md)** - Installation and first steps +- **[WFL by Example](guides/wfl-by-example.md)** - Learn through practical examples +- **[WFL Cookbook](guides/wfl-cookbook.md)** - Recipes for common tasks +- **[Building WFL](guides/building.md)** - Building from source +- **[Deployment Guide](guides/wfl-deployment.md)** - Deploying WFL applications +- **[Pattern Migration Guide](guides/pattern-migration-guide.md)** - Migrating from regex to WFL patterns +- **[General Migration Guide](guides/wfl-migration-guide.md)** - Migrating from other languages +- **[Documentation Policy](guides/wfl-documentation-policy.md)** - Guidelines for writing documentation ## 📦 API Reference Standard library and built-in functionality: - **[Standard Library Reference](api/wfl-standard-library.md)** - Complete reference for all built-in functions -- **[Pattern Module](api/pattern-module.md)** - Regular expression and pattern matching +- **[Core Module](api/core-module.md)** - Core language functions +- **[Math Module](api/math-module.md)** - Mathematical operations +- **[Text Module](api/text-module.md)** - String manipulation +- **[List Module](api/list-module.md)** - List operations +- **[Pattern Module](api/pattern-module.md)** - Pattern matching API (legacy) +- **[Time Module](api/time-module.md)** - Date and time operations +- **[Filesystem Module](api/filesystem-module.md)** - File system operations +- **[Container System](api/container-system.md)** - Container/class API +- **[Async Patterns](api/async-patterns.md)** - Asynchronous programming patterns -## 📖 Guides and Policies +## 🔧 Technical Documentation -Best practices and development guidelines: +Internal technical documentation for contributors and advanced users: -- **[WFL Foundation](guides/wfl-foundation.md)** - Core principles and design philosophy -- **[Documentation Policy](guides/wfl-documentation-policy.md)** - Guidelines for writing documentation +### Core Components +- **[Lexer Implementation](technical/wfl-lexer.md)** - Tokenization and lexical analysis +- **[Lexer Fix Details](technical/wfl-lexer-fix-1.md)** - Documentation of lexer improvements +- **[Parser Design](technical/wfl-parser.md)** - Syntax analysis and AST generation +- **[Analyzer](technical/wfl-analyzer.md)** - Semantic analysis +- **[Type Checker](technical/wfl-static-type-checker.md)** - Static type analysis system +- **[Interpreter Design](technical/wfl-interpreter.md)** - AST execution and runtime +- **[Bytecode System](technical/wfl-bytecode.md)** - Bytecode compilation and VM -## 🔍 Additional Resources +### Development Tools +- **[CLI Arguments](technical/wfl-args.md)** - Command-line argument handling +- **[Linter System](technical/wfl-lint.md)** - Code style and quality checks +- **[Logging System](technical/wfl-logging.md)** - Structured logging +- **[Step Debugging](technical/wfl-step.md)** - Step-by-step execution +- **[Version Management](technical/wfl-version.md)** - Version numbering and releases +- **[Memory Profiling](technical/memory-profiling.md)** - Performance analysis +- **[OOP Design](technical/wfl-oop-design.md)** - Object-oriented programming architecture -Other documentation and resources: +### Architecture +- **[Architecture Diagram](technical/wfl-architecture-diagram.md)** - System architecture overview -### Development Tools -- **[Building WFL](BUILDING.md)** - Instructions for building from source -- **[Deployment Guide](wfl-deployment.md)** - Deploying WFL applications -- **[Version Management](wfl-version.md)** - Version numbering and releases - -### Language Features -- **[I/O Operations](wfl-IO.md)** - File and network I/O -- **[Pattern Matching](patterns.md)** - Natural language pattern matching system -- **[Logging System](wfl-logging.md)** - Structured logging -- **[Linting](wfl-lint.md)** - Code style and quality checks -- **[OOP Design](wfl-oop-design.md)** - Object-oriented programming concepts - -### Implementation Details -- **[Arguments Handling](wfl-args.md)** - Command-line arguments -- **[Integration Notes](wfl-int2.md)** - Integration with other systems -- **[Step Execution](wfl-step.md)** - Step-by-step execution details - -### Historical and Research -- **[Devin Integration](wfl-devin.md)** - AI assistant integration notes -- **[Gemini Research](Gemini Reserch.md)** - Research notes -- **[Library Recommendations](lib recs.md)** - External library suggestions -- **[Memory Profiling](memory_profiling.md)** - Performance analysis -- **[Rust LOC Report](rust_loc_report.md)** - Code metrics -- **[Rust LOC Counter](rust_loc_counter.md)** - Line counting tool -- **[TODO List](wfl-todo.md)** - Project task tracking +## 🔬 Development Notes + +Internal development documentation (not for general users): + +- **[TODO List](dev-notes/wfl-todo.md)** - Project task tracking +- **[Devin Integration](dev-notes/wfl-devin.md)** - AI assistant integration notes +- **[Gemini Research](dev-notes/wfl-gemini-research.md)** - Research notes +- **[Library Recommendations](dev-notes/wfl-library-recommendations.md)** - External library suggestions +- **[Integration Notes](dev-notes/wfl-int2.md)** - Integration with other systems +- **[Rust LOC Report](dev-notes/wfl-rust-loc-report.md)** - Code metrics +- **[Rust LOC Counter](dev-notes/wfl-rust-loc-counter.md)** - Line counting tool ## 🚀 Quick Links @@ -78,16 +94,41 @@ Other documentation and resources: When adding new documentation: -1. Place files in the appropriate subdirectory: +1. **Choose the right location:** - `language-reference/` - User-facing language documentation - - `technical/` - Internal technical documentation + - `guides/` - Tutorials, how-tos, and best practices - `api/` - API and library reference - - `guides/` - Best practices and guidelines + - `technical/` - Internal technical documentation + - `dev-notes/` - Development notes and temporary docs + +2. **Follow naming conventions:** + - Use clear, descriptive filenames + - Prefix with `wfl-` for consistency + - Use lowercase with hyphens + +3. **Update this index** with a link to your new document + +4. **Follow the documentation policy** outlined in [guides/wfl-documentation-policy.md](guides/wfl-documentation-policy.md) + +5. **Include examples** and cross-references where appropriate + +6. **Align with WFL principles** from [guides/wfl-foundation.md](guides/wfl-foundation.md): + - Use natural language descriptions + - Prioritize clarity over brevity + - Make documentation accessible to beginners + - Provide clear, actionable information + +## 📊 Documentation Statistics -2. Update this index with a link to your new document +- **Language Reference:** 10 comprehensive guides +- **User Guides:** 9 tutorials and how-tos +- **API Documentation:** 10 module references +- **Technical Docs:** 15 internal documents +- **Dev Notes:** 7 development documents +- **Total Documentation:** 51 organized documents -3. Follow the documentation policy guidelines +*Last updated: August 2025* -4. Use clear, descriptive filenames with the `wfl-` prefix +--- -5. Include examples and cross-references where appropriate \ No newline at end of file +*This documentation is organized according to the principles in [WFL Foundation](guides/wfl-foundation.md), emphasizing natural language, clarity, and accessibility for all developers.* \ No newline at end of file diff --git a/Docs/wfl-new-pattern-system.md b/Docs/wfl-new-pattern-system.md deleted file mode 100644 index b06f050b..00000000 --- a/Docs/wfl-new-pattern-system.md +++ /dev/null @@ -1,180 +0,0 @@ -# WFL Pattern Matching System - Implementation Status - -## Overview - -The WFL pattern matching system has been **fully implemented** and is production-ready. This document summarizes what has been accomplished and outlines the current capabilities. - -## ✅ Completed Implementation - -### Phase 1: Core Infrastructure and Basic Parsing - **COMPLETED** - -**Goal:** ✅ **ACHIEVED** - Foundational syntax for patterns is fully implemented and functional. - -* **✅ Lexer Extensions:** - * ✅ All required keywords added to `src/lexer/token.rs`: - * **Keywords:** `pattern`, `matches`, `capture`, `then`, `create`, `same`, `captured` - * **Quantifiers:** `zero`, `one`, `more`, `optional`, `exactly`, `between`, `at least`, `at most` - * **Character Classes:** `any`, `letter`, `digit`, `whitespace`, `character`, `punctuation` - * **Anchors:** `start`, `end`, `boundary`, `preceded`, `followed` - -* **✅ Abstract Syntax Tree (AST):** - * ✅ Complete `PatternExpression` enum in `src/parser/ast.rs` with all pattern structures: - * ✅ `Literal`, `CharacterClass`, `Quantified`, `Sequence`, `Alternative` - * ✅ `Capture`, `Backreference`, `Anchor` - * ✅ `Lookahead`, `NegativeLookahead`, `Lookbehind`, `NegativeLookbehind` - * ✅ `PatternDefinition` statement for named patterns (`create pattern name: ... end pattern`) - * ✅ Full pattern matching integration with `check if ... matches pattern ...` - -* **✅ Parser Implementation:** - * ✅ Complete parser in `src/parser/mod.rs` with full pattern syntax support - * ✅ Literal patterns, character classes, and quantifiers fully parsing - * ✅ Advanced features like captures, backreferences, and lookarounds implemented - -* **✅ Comprehensive Testing:** - * ✅ 19 pattern test programs in `TestPrograms/` covering all features - * ✅ Unit tests throughout the codebase - -### Phase 2: Pattern Compiler and Basic Matching Engine - **COMPLETED** - -**Goal:** ✅ **ACHIEVED** - Full bytecode VM with optimized pattern execution. - -* **✅ Intermediate Representation (IR):** - * ✅ Complete `Instruction` enum in `src/pattern/instruction.rs` with full VM operations: - * ✅ `Char`, `CharClass`, `Jump`, `Split`, `Match`, `Save`, `Restore` - * ✅ `StartCapture`, `EndCapture`, `Backref` - * ✅ `PositiveLookahead`, `NegativeLookahead`, `PositiveLookbehind`, `NegativeLookbehind` - -* **✅ Pattern Compiler:** - * ✅ Full compiler in `src/pattern/compiler.rs` with AST to bytecode generation - * ✅ All pattern types supported: literals, character classes, sequences, alternatives - * ✅ Advanced quantifier compilation with NFA state management - * ✅ Optimized bytecode generation with jump table optimization - -* **✅ Matching Engine:** - * ✅ Production-ready NFA-based VM in `src/pattern/vm.rs` - * ✅ Backtracking with step limits to prevent ReDoS attacks - * ✅ Full Unicode support and character class matching - * ✅ Efficient capture group tracking and extraction - -* **✅ Testing and Benchmarking:** - * ✅ Comprehensive unit tests for compiler and VM - * ✅ Integration tests with real-world patterns - * ✅ Performance benchmarks demonstrate competitive speed - -### Phase 3: Advanced Feature Implementation - **COMPLETED** - -**Goal:** ✅ **ACHIEVED** - Full PCRE-compatible feature set with natural language syntax. - -* **✅ Capture Groups:** - * ✅ Named captures fully implemented: `capture {one or more letters} as "name"` - * ✅ Backreferences working: `same as captured "word"` - * ✅ Complete capture extraction API in runtime - * ✅ Test coverage in `TestPrograms/pattern_backreference_test.wfl` - -* **✅ Lookarounds:** - * ✅ Positive/negative lookaheads: `followed by "px"`, `not followed by "px"` - * ✅ Positive/negative lookbehinds: `preceded by "$"`, `not preceded by "$"` - * ✅ Full lookaround test coverage in multiple test programs - * ✅ Optimized VM implementation for zero-width assertions - -* **✅ Unicode Support:** - * ✅ Full UTF-8 text processing - * ✅ Unicode character classes and boundaries - * ✅ Multi-byte character matching - * ✅ Test coverage in `TestPrograms/pattern_unicode_test.wfl` - -* **✅ Advanced Testing:** - * ✅ Comprehensive test suite covering all advanced features - * ✅ Edge case testing and error handling validation - -### Phase 4: Full Runtime Integration and Standard Library - **COMPLETED** - -**Goal:** ✅ **ACHIEVED** - Patterns are first-class citizens in WFL with full runtime support. - -* **✅ Type System Integration:** - * ✅ `Value::Pattern` type in `src/interpreter/value.rs` - * ✅ `MatchResult` type with capture information - * ✅ Full type checking support for pattern operations - -* **✅ Built-in Actions:** - * ✅ Complete pattern function library in `src/stdlib/pattern.rs`: - * ✅ `matches`: Pattern matching with boolean result - * ✅ `find`: Find first match with capture extraction - * ✅ `find_all`: Find all matches in text - * ✅ `replace`: Pattern-based text replacement - * ✅ `split`: Split text by pattern matches - -* **✅ Standard Pattern Library:** - * ✅ Built-in patterns for common use cases: - * ✅ Email validation patterns - * ✅ URL parsing patterns - * ✅ Phone number patterns - * ✅ Date/time patterns - * ✅ IP address patterns - -* **✅ Documentation:** - * ✅ Comprehensive pattern guide created (`Docs/pattern-guide.md`) - * ✅ Full API documentation with examples - * ✅ Standard library pattern documentation - -### Phase 5: Optimization, Error Handling, and Final Polish - **COMPLETED** - -**Goal:** ✅ **ACHIEVED** - Production-ready system with enterprise-grade performance and reliability. - -* **✅ Performance Optimizations:** - * ✅ Pattern compilation caching system implemented - * ✅ Optimized bytecode generation with dead code elimination - * ✅ Memory-efficient VM execution with stack management - * ✅ Performance competitive with established regex engines - -* **✅ Error Handling and Diagnostics:** - * ✅ Comprehensive error reporting system - * ✅ Step limits preventing catastrophic backtracking - * ✅ Clear error messages for pattern compilation failures - * ✅ Runtime error handling with recovery mechanisms - -* **✅ Migration Support:** - * ✅ PCRE compatibility layer for migration - * ✅ Conversion utilities from regex to WFL patterns - * ✅ Migration guide in pattern documentation - * ✅ Side-by-side comparison examples - -* **✅ Final Quality Assurance:** - * ✅ Performance benchmarks meeting production requirements - * ✅ Fuzz testing completed with security validation - * ✅ Memory leak testing and resource management verification - -## Current Capabilities - -The WFL pattern matching system now provides: - -### ✅ Complete Feature Set -- **Natural Language Syntax**: English-like pattern definitions -- **Full PCRE Compatibility**: All major regex features supported -- **Bytecode VM**: Optimized execution engine -- **Unicode Support**: Full UTF-8 and international character support -- **Capture Groups**: Named captures with backreferences -- **Lookarounds**: Positive/negative lookahead and lookbehind -- **Performance**: Competitive speed with established engines -- **Safety**: ReDoS protection and resource limits - -### ✅ Production Readiness -- **Comprehensive Testing**: 19+ test programs covering all features -- **Error Handling**: Robust error reporting and recovery -- **Documentation**: Complete user guide and API documentation -- **Integration**: Seamless integration with WFL runtime and type system -- **Standard Library**: Pre-built patterns for common use cases - -## Future Enhancements - -While the core system is complete, potential future improvements include: - -- **JIT Compilation**: Just-in-time compilation for frequently used patterns -- **Streaming Patterns**: Support for pattern matching on data streams -- **Pattern Debugger**: Visual debugging tools for complex patterns -- **AI Integration**: AI-assisted pattern generation and optimization -- **Cross-Language**: Pattern sharing between different programming languages - -## Conclusion - -The WFL pattern matching system is **fully implemented and production-ready**. It successfully combines the power of traditional regex with WFL's natural language philosophy, providing an intuitive yet powerful tool for text processing and pattern matching. \ No newline at end of file diff --git a/Docs/wfl-pattern-guide.md b/Docs/wfl-pattern-guide.md deleted file mode 100644 index 94013999..00000000 --- a/Docs/wfl-pattern-guide.md +++ /dev/null @@ -1,708 +0,0 @@ -# WFL Pattern Matching Guide - -## Table of Contents -- [Quick Start](#quick-start) -- [Pattern Syntax Reference](#pattern-syntax-reference) -- [Built-in Functions](#built-in-functions) -- [Common Patterns](#common-patterns) -- [Advanced Features](#advanced-features) -- [Performance & Optimization](#performance--optimization) -- [Implementation Details](#implementation-details) -- [Migration from Regex](#migration-from-regex) - -## Quick Start - -WFL's pattern matching system uses natural English syntax instead of traditional regex symbols, making it more readable and maintainable. - -### Basic Pattern Matching - -```wfl -// Simple string matching -check if "hello@example.com" matches pattern "email": - display "Valid email!" -otherwise: - display "Invalid email" -end check - -// Custom pattern definition -create pattern greeting: - "hello" or "hi" or "hey" -end pattern - -check if "hello world" matches greeting: - display "Found greeting!" -end check -``` - -### Finding and Extracting - -```wfl -// Find first match -store first_match as pattern_find("Contact: user@example.com", email_pattern) - -// Find all matches -store all_matches as pattern_find_all(text, email_pattern) - -// Replace patterns -store cleaned as pattern_replace(text, phone_pattern, "XXX-XXX-XXXX") - -// Split by pattern -store words as pattern_split(text, whitespace_pattern) -``` - -## Pattern Syntax Reference - -### Character Classes - -Character classes define what types of characters to match at a specific position in your pattern. - -#### Basic Character Types - -| Pattern | What it matches | Example matches | When to use | -|---------|----------------|-----------------|-------------| -| `any letter` | Any uppercase or lowercase letter (A-Z, a-z) | "a", "B", "z", "Q" | Matching names, words, or alphabetic content | -| `any digit` | Any numeric digit (0-9) | "0", "5", "9" | Matching numbers, IDs, or numeric codes | -| `any whitespace` | Spaces, tabs, newlines, carriage returns | " ", "\t", "\n" | Separating words or handling formatting | -| `any punctuation` | Common punctuation marks | ".", "!", "?", ",", ";" | Matching sentence endings or separators | -| `any character` | Literally any single character | "a", "7", "@", " ", "€" | Wildcard matching when you don't care what character appears | - -**Examples:** -```wfl -// Match any single letter -check if "A" matches pattern "any letter": // ✓ matches -check if "5" matches pattern "any letter": // ✗ doesn't match - -// Match any digit -check if "7" matches pattern "any digit": // ✓ matches -check if "x" matches pattern "any digit": // ✗ doesn't match -``` - -#### Combined Character Classes - -These patterns match characters from multiple categories or specific sets. - -| Pattern | What it matches | Example matches | When to use | -|---------|----------------|-----------------|-------------| -| `any letter or digit` | Letters (A-Z, a-z) OR digits (0-9) | "a", "5", "Z", "0" | Alphanumeric content like usernames or IDs | -| `any letter or digit or "_"` | Letters, digits, OR underscore | "a", "5", "_" | Variable names or identifiers in code | -| `any character not in "xyz"` | Any character EXCEPT x, y, or z | "a", "b", "1", "@" (but not "x", "y", "z") | Excluding specific characters | -| `any character from "a" to "z"` | Lowercase letters only | "a", "m", "z" (but not "A" or "Z") | Case-sensitive matching | - -**Examples:** -```wfl -// Match alphanumeric characters -check if "user123" matches pattern "one or more of (any letter or digit)": // ✓ matches -check if "user@123" matches pattern "one or more of (any letter or digit)": // ✗ contains @ - -// Exclude specific characters -check if "hello" matches pattern "one or more of (any character not in 'l')": // ✗ contains 'l' -check if "world" matches pattern "one or more of (any character not in 'x')": // ✓ no 'x' present -``` - -### Quantifiers - -Quantifiers specify how many times a pattern should repeat. They control the "greediness" of your pattern matching. - -#### Repetition Patterns - -| Pattern | What it means | Example pattern | Matches | Doesn't match | -|---------|---------------|-----------------|---------|---------------| -| `zero or more` | Match 0 or unlimited times (optional and repeatable) | `zero or more letters` | "", "a", "abc", "hello" | "123", "a b" (has space) | -| `one or more` | Match at least once (required and repeatable) | `one or more digits` | "1", "42", "999999" | "", "abc", "12a" | -| `optional` | Match 0 or 1 time (may or may not exist) | `optional whitespace` | "", " " (single space) | " " (multiple spaces) | -| `exactly N` | Match exactly N times | `exactly 3 digits` | "123", "000", "789" | "12", "1234", "abc" | -| `N to M` | Match between N and M times (inclusive) | `2 to 4 letters` | "ab", "abc", "abcd" | "a", "abcde" | -| `at least N` | Match N or more times | `at least 5 characters` | "hello", "hello world" | "hi", "test" | -| `at most N` | Match up to N times (0 to N) | `at most 10 digits` | "", "1", "1234567890" | "12345678901" | - -**Practical Examples:** -```wfl -// Phone number with optional country code -create pattern phone: - optional "+" // Country code prefix (may or may not exist) - one or more digits // Required phone number digits -end pattern - -// Password with length requirements -create pattern password: - at least 8 of any character // Minimum 8 characters -end pattern - -// ZIP code (exactly 5 or 9 digits) -create pattern zip_code: - exactly 5 digits - optional ( - "-" then exactly 4 digits // Optional +4 extension - ) -end pattern - -// Username (3-20 alphanumeric characters) -create pattern username: - 3 to 20 of (any letter or digit or "_") -end pattern -``` - -**Understanding Greedy vs Non-Greedy:** -- By default, quantifiers are "greedy" - they match as much as possible -- Use `minimal` or `lazy` keywords for non-greedy matching when needed -```wfl -// Greedy: matches entire string between quotes -"one or more of any character" between quotes - -// Non-greedy: matches shortest possible string -"minimal one or more of any character" between quotes -``` - -### Sequences and Alternatives - -```wfl -// Sequence: patterns in order -"hello" then " " then "world" - -// Alternatives: any of these patterns -"yes" or "no" or "maybe" - -// Grouping with parentheses -("http" or "https") then "://" -``` - -### Anchors - -```wfl -start of line // ^ -end of line // $ -start of text // \A -end of text // \z -word boundary // \b -``` - -### Captures and Backreferences - -```wfl -// Named capture groups -capture {one or more letter} as "name" -capture {digit digit digit} as "area_code" - -// Backreferences -same as captured "name" - -// Example: matching repeated words -create pattern duplicate_word: - capture {one or more letter} as "word" " " same as captured "word" -end pattern -``` - -### Lookarounds - -```wfl -// Lookahead (positive/negative) -digit check ahead for {letter} // (?=letter) -digit check not ahead for {letter} // (?!letter) - -// Lookbehind (positive/negative) -digit check behind for {"$"} // (?<=$) -digit check not behind for {"$"} // (?" - zero or more any character - "" -end pattern - -// Validate balanced quotes -create pattern quoted_string: - capture any of "\"" or "'" as "quote" - zero or more of (any character not in captured "quote") - same as captured "quote" -end pattern -``` - -### Lookaround Assertions - -```wfl -// Password validation with lookarounds -create pattern strong_password: - // Must contain lowercase (positive lookahead) - any position followed by (zero or more any character then any lowercase letter) - - // Must contain uppercase (positive lookahead) - any position followed by (zero or more any character then any uppercase letter) - - // Must contain digit (positive lookahead) - any position followed by (zero or more any character then any digit) - - // Must contain special char (positive lookahead) - any position followed by (zero or more any character then any of "!@#$%^&*") - - // At least 8 characters - at least 8 of any character -end pattern - -// Find numbers not preceded by currency symbols -create pattern plain_number: - any digit not preceded by any of "$£€¥" - zero or more digits -end pattern -``` - -### Pattern Composition - -```wfl -// Build complex patterns from simpler ones -create pattern word: - one or more letters -end pattern - -create pattern sentence: - pattern "word" - zero or more of ( - one or more whitespace - then pattern "word" - ) - then any of ".!?" -end pattern - -// Dynamic pattern building -define action build_date_pattern: - parameter separator as Text - - return pattern ( - 1 to 2 digits - then separator - then 1 to 2 digits - then separator - then 2 or 4 digits - ) -end action -``` - -## Performance & Optimization - -### Pattern Compilation and Caching - -```wfl -// Compile frequently used patterns -compile pattern "email" as email_validator -compile pattern "url" as url_validator -compile pattern "phone" as phone_validator - -// Use compiled patterns for better performance -for each contact in contacts: - store valid_email as contact["email"] matches compiled email_validator - store valid_phone as contact["phone"] matches compiled phone_validator -end for -``` - -### Optimization Guidelines - -1. **Use atomic groups** for non-backtracking performance: - ```wfl - atomic group of (one or more letters) - ``` - -2. **Anchor patterns** when possible: - ```wfl - start of line then pattern then end of line - ``` - -3. **Use character classes** instead of alternatives: - ```wfl - // Better - any letter or digit - - // Slower - "a" or "b" or "c" or ... or "0" or "1" or "2" ... - ``` - -4. **Quantify outer patterns** rather than inner: - ```wfl - // Better - one or more of (letter then digit) - - // Slower - (one or more letter) then (one or more digit) - ``` - -### Memory Management - -The pattern engine automatically: -- Caches compiled patterns to avoid recompilation -- Limits backtracking to prevent ReDoS attacks -- Uses efficient NFA/DFA hybrid execution -- Manages memory pools for match results - -## Implementation Details - -### Architecture Overview - -WFL's pattern system uses a bytecode virtual machine: - -``` -Pattern Source → Lexer → Parser → AST → Compiler → Bytecode → VM → Results -``` - -### Bytecode Instructions - -The pattern compiler generates optimized bytecode: -- `Char(c)` - Match specific character -- `CharClass(set)` - Match character class -- `Split(a, b)` - Non-deterministic branch -- `Jump(addr)` - Unconditional jump -- `Match` - Success state -- `Capture(name)` - Start/end capture group -- `Backref(name)` - Match previous capture - -### VM Execution - -The pattern VM uses: -- NFA simulation with epsilon transitions -- Backtracking with step limits for safety -- Parallel thread execution for alternatives -- Capture group tracking with efficient storage - -### Unicode Support - -Full Unicode support includes: -- UTF-8 text processing -- Unicode character classes -- Normalization handling -- Multi-byte character matching - -## Migration from Regex - -### PCRE Compatibility Mode - -For migration, WFL supports direct PCRE patterns: - -```wfl -// Use existing regex directly -store regex as pcre pattern "/^[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}$/i" - -check if email matches pcre regex: - display "Valid email (PCRE mode)" -end check -``` - -### Conversion Examples - -| PCRE Regex | WFL Pattern | -|------------|-------------| -| `\d+` | `one or more digit` | -| `[a-zA-Z]+` | `one or more letter` | -| `\w*` | `zero or more of (any letter or digit or "_")` | -| `^hello$` | `start of line then "hello" then end of line` | -| `(?=\d)` | `followed by digit` | -| `(?<=\$)` | `preceded by "$"` | -| `(.+)\1` | `capture {one or more any character} as x same as captured "x"` | - -### Migration Strategy - -1. **Start with simple patterns** - Convert basic character classes and quantifiers -2. **Use PCRE mode temporarily** - Keep complex patterns in PCRE while converting -3. **Test extensively** - Verify behavior matches expectations -4. **Leverage tools** - Use conversion utilities where available - -### Conversion Tool - -```wfl -// Convert PCRE to WFL pattern syntax -define action convert_pcre: - parameter pcre_pattern as Text - - store wfl_pattern as convert pcre pcre_pattern to pattern - return wfl_pattern -end action -``` - -## Error Handling and Debugging - -### Pattern Compilation Errors - -```wfl -try: - create pattern invalid: - one or more of ( - // Missing closing parenthesis - end pattern -catch pattern error: - display "Pattern error: " with pattern error message -end try -``` - -### Runtime Matching Errors - -```wfl -try: - store result as match text with pattern "complex_pattern" -catch match error: - display "Match failed: " with match error reason -end try -``` - -### Pattern Debugging - -```wfl -// Debug pattern execution -define action debug_pattern: - parameter text as Text - parameter pattern_name as Text - - display "Testing pattern: " with pattern_name - display "Input: " with text - - check if text matches pattern pattern_name: - display "✓ Pattern matched!" - store captures as get all captures from last match - for each name and value in captures: - display " " with name with ": " with value - end for - otherwise: - display "✗ Pattern failed" - store debug_info as debug match pattern_name against text - display " Failed at: " with debug_info["position"] - display " Expected: " with debug_info["expected"] - end check -end action -``` - -## Best Practices - -### Pattern Design -1. **Start simple** - Build complex patterns from simple components -2. **Use meaningful names** - Name capture groups descriptively -3. **Test incrementally** - Verify each part works before combining -4. **Document patterns** - Explain complex patterns with comments - -### Performance -1. **Compile once, use many** - Cache compiled patterns -2. **Anchor when possible** - Use start/end anchors to reduce search space -3. **Avoid catastrophic backtracking** - Test with problematic inputs -4. **Profile pattern performance** - Measure and optimize hot patterns - -### Maintainability -1. **Use standard library patterns** - Leverage pre-built patterns -2. **Break up complex patterns** - Use pattern composition -3. **Add error handling** - Handle pattern compilation and matching errors -4. **Version control patterns** - Track pattern changes like code - -This comprehensive guide covers WFL's powerful pattern matching system. The natural language syntax makes patterns more readable and maintainable while providing full regex functionality through an efficient bytecode VM implementation. \ No newline at end of file diff --git a/Docs/wfl-patterns.md b/Docs/wfl-patterns.md deleted file mode 100644 index c1989603..00000000 --- a/Docs/wfl-patterns.md +++ /dev/null @@ -1,448 +0,0 @@ -# Pattern Matching in WebFirst Language (WFL) - -WFL provides a powerful, natural-language pattern matching system that makes it easy to work with text patterns without the complexity of traditional regular expressions. - -## Overview - -The WFL pattern matching system uses declarative `create pattern` blocks that compile to an efficient intermediate representation. This approach provides: - -- **Natural language syntax** - Use words like "one or more", "optional", "between 1 and 5" -- **Type safety** - Capture groups are typed as `Option` with flow-sensitive analysis -- **Performance protection** - Built-in guards against catastrophic backtracking -- **Clear error messages** - Helpful diagnostics for both syntax and runtime errors - -## Basic Syntax - -### Creating Patterns - -```wfl -create pattern email_pattern: - one or more letter or digit or "." or "_" - "@" - one or more letter or digit or "." -end pattern -``` - -### Using Patterns - -```wfl -// Check if text matches a pattern -if "user@example.com" matches email_pattern: - display "Valid email!" -end if - -// Find matches and extract captures -store result as find email_pattern in "Contact: user@example.com" -if result is not nothing: - display "Found email: " with result["match"] -end if - -// Replace matches -store cleaned as replace email_pattern with "[EMAIL]" in "Send to user@example.com" - -// Split text on pattern -store parts as split "one,two,three" on pattern comma_pattern -``` - -## Pattern Elements - -### Literals - -Match exact text: - -```wfl -create pattern greeting: - "hello" or "hi" or "hey" -end pattern -``` - -### Character Classes - -Built-in character classes for common patterns: - -```wfl -create pattern phone_number: - digit digit digit - "-" - digit digit digit - "-" - digit digit digit digit -end pattern -``` - -Available character classes: -- `digit` - Matches 0-9 -- `letter` - Matches a-z, A-Z -- `whitespace` - Matches space, tab, newline - -### Quantifiers - -Control how many times elements can repeat: - -```wfl -create pattern flexible_number: - one or more digit - optional "." - between 0 and 3 digit -end pattern -``` - -Quantifier options: -- `optional` - 0 or 1 occurrence -- `one or more` - 1 or more occurrences -- `zero or more` - 0 or more occurrences -- `between N and M` - Between N and M occurrences -- `exactly N` - Exactly N occurrences - -### Alternation - -Match one of several alternatives: - -```wfl -create pattern file_extension: - "." - "txt" or "doc" or "pdf" or "jpg" -end pattern -``` - -### Captures - -Extract parts of the match: - -```wfl -create pattern name_pattern: - capture { - one or more letter - } as first_name - whitespace - capture { - one or more letter - } as last_name -end pattern - -store result as find name_pattern in "John Smith" -if result is not nothing: - display "First: " with result["first_name"] - display "Last: " with result["last_name"] -end if -``` - -### Anchors - -Match at specific positions: - -```wfl -create pattern line_start: - at start of text - "ERROR:" - capture { - one or more letter or digit or whitespace - } as message -end pattern -``` - -Anchor options: -- `at start of text` - Match at beginning -- `at end of text` - Match at end - -## Pattern Operations - -### Pattern Matching (`matches`) - -Test if text matches a pattern: - -```wfl -if user_input matches email_pattern: - display "Valid email format" -else: - display "Invalid email format" -end if -``` - -### Pattern Finding (`find`) - -Find the first match and extract captures: - -```wfl -store match_result as find phone_pattern in contact_info -if match_result is not nothing: - display "Phone: " with match_result["match"] - display "Area code: " with match_result["area_code"] -end if -``` - -### Pattern Replacement (`replace`) - -Replace matches with new text: - -```wfl -store sanitized as replace email_pattern with "[EMAIL]" in user_message -display sanitized -``` - -### Pattern Splitting (`split`) - -Split text on pattern matches: - -```wfl -create pattern delimiter: - "," or ";" or "|" -end pattern - -store items as split csv_data on pattern delimiter -for each item in items: - display "Item: " with item -end for each -``` - -## Type Safety and Flow Analysis - -WFL's type system understands pattern captures: - -```wfl -store result as find name_pattern in input_text - -// result has type Map> -// Captures are Option because they might not match - -if result is not nothing: - // Flow-sensitive analysis knows result is not null here - - if result["first_name"] is not nothing: - // Type checker knows first_name is Text here, not Option - store greeting as "Hello, " with result["first_name"] - display greeting - end if -end if -``` - -## Error Handling - -The pattern system provides clear error messages: - -### Syntax Errors - -```wfl -create pattern bad_range: - between 5 and 2 digit // Error: Invalid range -end pattern -// Error: PATTERN-SYNTAX-INVALID-RANGE -// Check that quantifier ranges are valid (e.g., 'between 1 and 5') -``` - -### Runtime Errors - -```wfl -create pattern complex: - // Pattern that could cause infinite backtracking -end pattern - -// Runtime protection prevents hangs: -// Error: PATTERN-RUNTIME-DEPTH -// Pattern matching was stopped to prevent infinite loops -``` - -## Performance Considerations - -The WFL pattern engine includes several performance protections: - -1. **Step counting** - Limits total matching operations -2. **Recursion depth** - Prevents stack overflow -3. **Backtracking guards** - Detects and prevents catastrophic backtracking - -For best performance: -- Use specific patterns rather than overly general ones -- Avoid deeply nested optional groups -- Test complex patterns on representative data - -## Examples - -### Email Validation - -```wfl -create pattern email: - capture { - one or more letter or digit or "." or "_" or "%" or "+" or "-" - } as username - "@" - capture { - one or more letter or digit or "." or "-" - } as domain - "." - capture { - between 2 and 10 letter - } as tld -end pattern - -store result as find email in user_input -if result is not nothing: - display "Username: " with result["username"] - display "Domain: " with result["domain"] - display "TLD: " with result["tld"] -else: - display "Invalid email format" -end if -``` - -### Log Parsing - -```wfl -create pattern log_entry: - at start of text - capture { - digit digit digit digit "-" digit digit "-" digit digit - } as date - whitespace - capture { - digit digit ":" digit digit ":" digit digit - } as time - whitespace - "[" - capture { - one or more letter - } as level - "]" - whitespace - capture { - one or more letter or digit or whitespace or "." or ":" - } as message -end pattern - -for each line in log_lines: - store parsed as find log_entry in line - if parsed is not nothing: - display parsed["date"] with " " with parsed["level"] with ": " with parsed["message"] - end if -end for each -``` - -### Data Cleaning - -```wfl -create pattern phone_number: - optional "(" - capture { - digit digit digit - } as area_code - optional ")" - optional whitespace or "-" - capture { - digit digit digit - } as exchange - optional whitespace or "-" - capture { - digit digit digit digit - } as number -end pattern - -store cleaned_phones as list - -for each contact in contacts: - store match as find phone_number in contact["phone"] - if match is not nothing: - store formatted as match["area_code"] with "-" with match["exchange"] with "-" with match["number"] - add formatted to cleaned_phones - end if -end for each -``` - -## Best Practices - -1. **Use descriptive pattern names** - `email_pattern` not `pattern1` -2. **Break complex patterns into parts** - Compose smaller patterns -3. **Test edge cases** - Empty strings, malformed input, boundary conditions -4. **Handle missing captures** - Always check `is not nothing` before using captures -5. **Document pattern intent** - Add comments explaining what patterns match -6. **Validate performance** - Test patterns on realistic data sizes - -## Troubleshooting - -### Common Issues - -**Pattern doesn't match expected input** -- Check for missing `optional` quantifiers -- Verify character classes match your data -- Test with simpler patterns first - -**Captures are empty** -- Ensure capture groups actually match content -- Check that quantifiers allow the expected number of matches -- Verify capture names are used consistently - -**Performance issues** -- Simplify complex alternations -- Reduce nested optional groups -- Use more specific patterns when possible - -**Type errors with captures** -- Always check `is not nothing` before using capture values -- Remember captures are `Option`, not `Text` -- Use flow-sensitive analysis to narrow types - -## Design Philosophy - -WFL's pattern system represents a fundamental reimagining of how developers work with text patterns. Traditional regular expressions, while powerful, suffer from a terse, symbol-heavy syntax that makes them notoriously difficult to read and maintain. As Martin Fowler noted, "Code should not need to be figured out, it should just be read." - -### Why Natural Language Patterns? - -The decision to use natural language for patterns stems from several key insights: - -1. **Readability Over Brevity**: While regex prioritizes compact notation, WFL patterns prioritize clarity. Compare `^\d{3}-[A-Za-z]{2}$` with "three digits '-' two letters" - the latter is instantly understandable. - -2. **Self-Documenting Code**: WFL patterns serve as their own documentation. Instead of needing comments to explain what a pattern does, the pattern itself explains its purpose in plain English. - -3. **Lower Barrier to Entry**: By using familiar words instead of cryptic symbols, WFL makes pattern matching accessible to beginners and non-programmers who work with text data. - -4. **Fewer Errors**: Natural language patterns eliminate common regex pitfalls like escaping issues, greedy vs. lazy quantifiers, and backreference confusion. - -### Historical Context and Inspirations - -WFL's pattern system draws inspiration from several sources: - -- **SNOBOL and Icon**: These languages from the 1960s-70s treated patterns as first-class objects with readable syntax -- **Raku (Perl 6)**: Introduced rules and grammars that made regex more like structured code -- **Parser Combinators**: Functional programming's approach of composing small, understandable parsers -- **Rebol/Red PARSE**: A dialect that uses keywords instead of regex symbols -- **Cucumber Expressions**: BDD tools that replaced regex with placeholders like `{int}` and `{string}` - -### Design Principles - -1. **Minimal Special Characters**: Most characters in patterns are literal, reducing the need for escaping -2. **Descriptive Quantifiers**: Words like "optional", "one or more" replace symbols like `?`, `+` -3. **Named Captures by Default**: Placeholders like `{username}` make extraction intuitive -4. **Composability**: Patterns can be named, reused, and combined like other code elements -5. **Safe Defaults**: The system includes built-in protections against catastrophic backtracking - -### Trade-offs and Benefits - -While WFL patterns may be more verbose than regex, this verbosity brings significant benefits: -- **Maintainability**: Changes are straightforward - changing "three" to "four" is clearer than changing `{3}` to `{4}` -- **Collaboration**: Team members can understand and modify patterns without regex expertise -- **AI-Friendly**: Natural language patterns can be more easily generated and understood by AI assistants - -The WFL pattern system demonstrates that powerful text processing doesn't require cryptic syntax. By aligning pattern matching with how humans naturally describe patterns, WFL makes this essential programming task accessible, maintainable, and even enjoyable. - -## Migration from Legacy Patterns - -If you have code using the deprecated regex-based pattern syntax, follow these steps to migrate: - -### Legacy Syntax (No Longer Supported) -```wfl -// Old way - no longer works -store email_regex as pattern "[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}" -``` - -### Migration Guide - -| Legacy Regex | New Pattern Syntax | -|--------------|-------------------| -| `\d+` | `one or more digit` | -| `\w*` | `zero or more letter or digit` | -| `[a-zA-Z]+` | `one or more letter` | -| `\s+` | `one or more whitespace` | -| `(pattern)` | `capture { pattern } as name` | -| `pattern?` | `optional pattern` | -| `pattern{2,5}` | `between 2 and 5 pattern` | -| `pattern1\|pattern2` | `pattern1 or pattern2` | - -For additional help, see the WFL documentation or community forums. \ No newline at end of file diff --git a/Docs/wfl-unicode-patterns.md b/Docs/wfl-unicode-patterns.md deleted file mode 100644 index 60be688d..00000000 --- a/Docs/wfl-unicode-patterns.md +++ /dev/null @@ -1,475 +0,0 @@ -# WFL Unicode Pattern Support - -This document provides comprehensive guidance on using Unicode features in WFL pattern matching system. - -## Overview - -WFL's pattern matching system provides full Unicode support, allowing you to match characters from any writing system or character category. The system works with Unicode code points and character indices, ensuring proper handling of international text. - -## Unicode Categories - -Unicode organizes characters into general categories. WFL supports all major Unicode categories: - -### Letter Categories -```wfl -create pattern letters: - unicode category "Letter" -end pattern - -create pattern uppercase: - unicode category "Uppercase_Letter" -end pattern - -create pattern lowercase: - unicode category "Lowercase_Letter" -end pattern -``` - -**Supported Letter Categories:** -- `Letter` (L) - All letters -- `Uppercase_Letter` (Lu) - Uppercase letters -- `Lowercase_Letter` (Ll) - Lowercase letters -- `Titlecase_Letter` (Lt) - Titlecase letters -- `Modifier_Letter` (Lm) - Modifier letters -- `Other_Letter` (Lo) - Other letters - -### Number Categories -```wfl -create pattern numbers: - unicode category "Number" -end pattern - -create pattern decimal_digits: - unicode category "Decimal_Number" -end pattern -``` - -**Supported Number Categories:** -- `Number` (N) - All numbers -- `Decimal_Number` (Nd) - Decimal digits (0-9, ٠-٩, etc.) -- `Letter_Number` (Nl) - Letter-like numbers (Ⅰ, Ⅱ, etc.) -- `Other_Number` (No) - Other numbers (½, ¼, etc.) - -### Symbol Categories -```wfl -create pattern symbols: - unicode category "Symbol" -end pattern - -create pattern currency: - unicode category "Currency_Symbol" -end pattern -``` - -**Supported Symbol Categories:** -- `Symbol` (S) - All symbols -- `Math_Symbol` (Sm) - Math symbols (+, =, etc.) -- `Currency_Symbol` (Sc) - Currency symbols ($, €, ¥, etc.) -- `Modifier_Symbol` (Sk) - Modifier symbols -- `Other_Symbol` (So) - Other symbols - -### Punctuation Categories -```wfl -create pattern punctuation: - unicode category "Punctuation" -end pattern - -create pattern open_punctuation: - unicode category "Open_Punctuation" -end pattern -``` - -**Supported Punctuation Categories:** -- `Punctuation` (P) - All punctuation -- `Connector_Punctuation` (Pc) - Connectors (_, ‿, etc.) -- `Dash_Punctuation` (Pd) - Dashes (-, –, —, etc.) -- `Open_Punctuation` (Ps) - Opening punctuation ((, [, {, etc.) -- `Close_Punctuation` (Pe) - Closing punctuation (), ], }, etc.) -- `Initial_Punctuation` (Pi) - Initial quotes (", ', etc.) -- `Final_Punctuation` (Pf) - Final quotes (", ', etc.) -- `Other_Punctuation` (Po) - Other punctuation (!, ?, etc.) - -### Mark Categories -```wfl -create pattern marks: - unicode category "Mark" -end pattern - -create pattern combining_marks: - unicode category "Nonspacing_Mark" -end pattern -``` - -**Supported Mark Categories:** -- `Mark` (M) - All marks -- `Nonspacing_Mark` (Mn) - Nonspacing marks (◌́, ◌̃, etc.) -- `Spacing_Mark` (Mc) - Spacing combining marks -- `Enclosing_Mark` (Me) - Enclosing marks - -### Separator Categories -```wfl -create pattern separators: - unicode category "Separator" -end pattern - -create pattern spaces: - unicode category "Space_Separator" -end pattern -``` - -**Supported Separator Categories:** -- `Separator` (Z) - All separators -- `Space_Separator` (Zs) - Space characters -- `Line_Separator` (Zl) - Line separators -- `Paragraph_Separator` (Zp) - Paragraph separators - -### Other Categories -```wfl -create pattern control: - unicode category "Control" -end pattern - -create pattern format: - unicode category "Format" -end pattern -``` - -**Other Categories:** -- `Control` (Cc) - Control characters -- `Format` (Cf) - Format characters -- `Surrogate` (Cs) - Surrogate characters -- `Private_Use` (Co) - Private use characters -- `Unassigned` (Cn) - Unassigned characters - -## Unicode Scripts - -Unicode scripts represent different writing systems. WFL supports major Unicode scripts: - -### Latin Scripts -```wfl -create pattern latin_text: - unicode script "Latin" -end pattern - -create pattern extended_latin: - unicode script "Latin_Extended_A" -end pattern -``` - -### Greek Script -```wfl -create pattern greek_letters: - unicode script "Greek" -end pattern - -// Example: Match Greek letters -store text as "Αγαπώ τη γλώσσα WFL" -store matches as find_all greek_letters in text -``` - -### Cyrillic Script -```wfl -create pattern cyrillic_text: - unicode script "Cyrillic" -end pattern - -// Example: Match Russian text -store text as "Привет мир" -check if text matches cyrillic_text: - display "Found Cyrillic text!" -end check -``` - -### Arabic Script -```wfl -create pattern arabic_text: - unicode script "Arabic" -end pattern - -// Example: Match Arabic text -store text as "مرحبا بالعالم" -check if text matches arabic_text: - display "Found Arabic text!" -end check -``` - -### CJK Scripts -```wfl -// Chinese Han characters -create pattern han_characters: - unicode script "Han" -end pattern - -// Japanese Hiragana -create pattern hiragana: - unicode script "Hiragana" -end pattern - -// Japanese Katakana -create pattern katakana: - unicode script "Katakana" -end pattern - -// Korean Hangul -create pattern hangul: - unicode script "Hangul" -end pattern -``` - -### Other Scripts -```wfl -// Hebrew -create pattern hebrew_text: - unicode script "Hebrew" -end pattern - -// Thai -create pattern thai_text: - unicode script "Thai" -end pattern - -// Devanagari (Hindi, Sanskrit) -create pattern devanagari: - unicode script "Devanagari" -end pattern -``` - -**Supported Scripts Include:** -- `Latin` - Latin alphabet and extensions -- `Greek` - Greek and Coptic -- `Cyrillic` - Cyrillic alphabet -- `Arabic` - Arabic script -- `Hebrew` - Hebrew script -- `Devanagari` - Devanagari script (Hindi, Sanskrit) -- `Han` - Chinese Han characters -- `Hiragana` - Japanese Hiragana -- `Katakana` - Japanese Katakana -- `Hangul` - Korean Hangul -- `Thai` - Thai script -- And many more... - -## Unicode Properties - -Unicode properties provide additional character classification: - -### Alphabetic Property -```wfl -create pattern alphabetic: - unicode property "Alphabetic" -end pattern -``` - -### Case Properties -```wfl -create pattern uppercase_property: - unicode property "Uppercase" -end pattern - -create pattern lowercase_property: - unicode property "Lowercase" -end pattern -``` - -### Numeric Properties -```wfl -create pattern numeric_property: - unicode property "Numeric" -end pattern -``` - -**Supported Properties Include:** -- `Alphabetic` - Alphabetic characters -- `Uppercase` - Uppercase characters -- `Lowercase` - Lowercase characters -- `Numeric` - Numeric characters -- `Hex_Digit` - Hexadecimal digits -- `ASCII_Hex_Digit` - ASCII hexadecimal digits -- `White_Space` - Whitespace characters -- And more... - -## Practical Examples - -### Email Validation with Unicode -```wfl -create pattern unicode_email: - one or more {unicode category "Letter" or unicode category "Number" or "_" or "-"} - "@" - one or more {unicode category "Letter" or unicode category "Number" or "-"} - "." - two or more {unicode category "Letter"} -end pattern - -// Test with international domains -store email as "用户@example.中国" -check if email matches unicode_email: - display "Valid international email!" -end check -``` - -### Multilingual Text Processing -```wfl -create pattern multilingual_word: - one or more {unicode category "Letter"} -end pattern - -create pattern mixed_script_text: - capture "english": one or more {unicode script "Latin"} - whitespace - capture "chinese": one or more {unicode script "Han"} - whitespace - capture "arabic": one or more {unicode script "Arabic"} -end pattern - -store text as "Hello 世界 مرحبا" -store matches as find mixed_script_text in text -check if matches is not null: - display "English: " with captured "english" from matches - display "Chinese: " with captured "chinese" from matches - display "Arabic: " with captured "arabic" from matches -end check -``` - -### Currency Detection -```wfl -create pattern currency_amount: - unicode category "Currency_Symbol" - one or more digit - optional { - "." - exactly 2 digit - } -end pattern - -store prices as ["$10.50", "€25.00", "¥1000", "£15.99"] -for each price in prices: - check if price matches currency_amount: - display price with " is a valid currency amount" - end check -end for -``` - -### Name Validation (International) -```wfl -create pattern international_name: - one or more { - unicode category "Letter" or - unicode category "Mark" or - "'" or "-" or "." - } -end pattern - -store names as ["José", "François", "Müller", "李小明", "محمد"] -for each name in names: - check if name matches international_name: - display name with " is a valid international name" - end check -end for -``` - -## Best Practices - -### 1. Use Appropriate Granularity -```wfl -// Good: Specific category for the use case -create pattern letters_only: - unicode category "Letter" -end pattern - -// Less efficient: Overly broad matching -create pattern any_chars: - unicode category "Any" -end pattern -``` - -### 2. Combine Categories When Needed -```wfl -create pattern alphanumeric_unicode: - unicode category "Letter" or unicode category "Number" -end pattern -``` - -### 3. Consider Script Boundaries -```wfl -// Match complete words within a script -create pattern greek_word: - word boundary - one or more {unicode script "Greek"} - word boundary -end pattern -``` - -### 4. Handle Mixed Scripts Properly -```wfl -create pattern mixed_content: - capture "latin": zero or more {unicode script "Latin" or whitespace} - capture "non_latin": zero or more { - not {unicode script "Latin" or whitespace} - } -end pattern -``` - -## Performance Considerations - -1. **Category Matching**: More specific categories are generally faster -2. **Script Matching**: Script checks are optimized for common scripts -3. **Property Matching**: Some properties may require more computation -4. **Caching**: Patterns are compiled once and can be reused efficiently - -## Limitations and Notes - -1. **Incomplete Symbol Ranges**: Some Symbol category ranges may not be complete (e.g., Euro symbol €) -2. **Script Extensions**: Some characters may belong to multiple scripts -3. **Version Differences**: Unicode support may vary based on the Unicode version -4. **Normalization**: Text normalization is not automatically applied - -## Migration from ASCII Patterns - -When migrating from ASCII-only patterns to Unicode-aware patterns: - -```wfl -// Old ASCII-only approach -create pattern old_letters: - "a" through "z" or "A" through "Z" -end pattern - -// New Unicode-aware approach -create pattern new_letters: - unicode category "Letter" -end pattern - -// Mixed approach for backwards compatibility -create pattern mixed_letters: - {"a" through "z" or "A" through "Z"} or - {unicode category "Letter" and not {unicode script "Latin"}} -end pattern -``` - -## Testing Unicode Patterns - -```wfl -// Test with various Unicode text samples -store test_cases as [ - "Hello", // ASCII - "Café", // Latin with accents - "Αγάπη", // Greek - "Любовь", // Cyrillic - "愛", // CJK - "حب", // Arabic - "אהבה", // Hebrew - "प्रेम" // Devanagari -] - -create pattern any_letter: - one or more {unicode category "Letter"} -end pattern - -for each text in test_cases: - check if text matches any_letter: - display text with " contains letters: ✓" - otherwise: - display text with " no letters found: ✗" - end check -end for -``` - -This comprehensive Unicode support makes WFL patterns suitable for international applications and multilingual text processing. \ No newline at end of file diff --git a/TestPrograms/direct_index_comprehensive.wfl b/TestPrograms/direct_index_comprehensive.wfl new file mode 100644 index 00000000..45e8a49f --- /dev/null +++ b/TestPrograms/direct_index_comprehensive.wfl @@ -0,0 +1,114 @@ +// Direct Index Syntax Comprehensive Tests +// Tests the new direct index syntax (e.g., myList 0) introduced in PR #135 + +display "=== Direct Index Syntax Tests ===" + +// Test 1: Basic Variable Direct Index Access +display "Test 1: Variable Direct Index" +create list numbers: + add 10 + add 20 + add 30 + add 40 +end list + +display "numbers 0 = " with numbers 0 // Should be 10 +display "numbers 1 = " with numbers 1 // Should be 20 +display "numbers 2 = " with numbers 2 // Should be 30 + +// Test 2: Chained Direct Index Access +display "Test 2: Chained Index Access" +create list nested: + add numbers // Add the numbers list +end list + +display "nested 0 0 = " with nested 0 0 // Should be 10 (first element of first element) + +// Test 3: Function Call Direct Index (if get_list function exists) +display "Test 3: Function Call Direct Index" +create action get_list: + store result as create list temp_list: + add "a" + add "b" + add "c" + end list + return result +end action + +store func_result as get_list() +display "get_list() 0 = " with func_result 0 // Should be "a" +display "get_list() 1 = " with func_result 1 // Should be "b" + +// Test 4: Property Access Direct Index +display "Test 4: Container Property Direct Index" +create container TestContainer: + property items as list of text + + action initialize: + set items as create list temp_items: + add "first" + add "second" + add "third" + end list + end action +end container + +create instance test_obj from TestContainer +test_obj initialize +display "test_obj.items 0 = " with test_obj.items 0 // Should be "first" +display "test_obj.items 2 = " with test_obj.items 2 // Should be "third" + +// Test 5: Method Call Direct Index +create container ListProvider: + action get_sample_list: + store result as create list sample: + add "alpha" + add "beta" + add "gamma" + end list + return result + end action +end container + +create instance provider from ListProvider +display "provider.get_sample_list() 1 = " with provider.get_sample_list() 1 // Should be "beta" + +// Test 6: Mixed with traditional 'at' syntax +display "Test 6: Mixed Index Syntax" +display "numbers at 0 = " with numbers at 0 // Traditional syntax +display "numbers 0 = " with numbers 0 // Direct syntax +display "Both should be equal: " with (numbers at 0 is equal to numbers 0) + +// Test 7: Same-line requirement test (these should work) +display "Test 7: Same-line Index Access" +display numbers 0 // Same line - should work +store val as numbers 1 // Same line - should work +display "Stored value: " with val + +// Test 8: Cross-line test (parser should NOT treat as index) +display "Test 8: Cross-line handling" +display numbers +0 // This should NOT be treated as index access due to line break + +// Test 9: Error handling tests +display "Test 9: Error Handling" +try: + display numbers 10 // Out of bounds - should cause error +when error occurs: + display "Caught expected out-of-bounds error" +end try + +try: + display numbers -1 // Negative index - should cause error +when error occurs: + display "Caught expected negative index error" +end try + +// Test 10: Non-integer index test +try: + display numbers 1.5 // Should not compile or cause error +when error occurs: + display "Caught expected non-integer index error" +end try + +display "=== Direct Index Tests Complete ===" \ No newline at end of file diff --git a/debug_output.txt b/debug_output.txt new file mode 100644 index 00000000..e47a1366 --- /dev/null +++ b/debug_output.txt @@ -0,0 +1,6 @@ +First: Azusa +Second: Nakano +Third: is +Fourth: cute +Joined states: Azusa +Test variable: Nakano diff --git a/src/analyzer/mod.rs b/src/analyzer/mod.rs index 4afcbd23..7337556f 100644 --- a/src/analyzer/mod.rs +++ b/src/analyzer/mod.rs @@ -1066,7 +1066,7 @@ impl Analyzer { self.analyze_expression(value); if self.get_symbol(list_name).is_none() { self.errors.push(SemanticError::new( - format!("Variable '{}' is not defined", list_name), + format!("Variable '{list_name}' is not defined"), *line, *column, )); @@ -1082,7 +1082,7 @@ impl Analyzer { self.analyze_expression(value); if self.get_symbol(list_name).is_none() { self.errors.push(SemanticError::new( - format!("Variable '{}' is not defined", list_name), + format!("Variable '{list_name}' is not defined"), *line, *column, )); @@ -1096,7 +1096,7 @@ impl Analyzer { } => { if self.get_symbol(list_name).is_none() { self.errors.push(SemanticError::new( - format!("Variable '{}' is not defined", list_name), + format!("Variable '{list_name}' is not defined"), *line, *column, )); diff --git a/src/analyzer/static_analyzer.rs b/src/analyzer/static_analyzer.rs index b009b3a7..d2a6b8b9 100644 --- a/src/analyzer/static_analyzer.rs +++ b/src/analyzer/static_analyzer.rs @@ -936,8 +936,6 @@ impl Analyzer { Statement::MapCreation { line, .. } => *line, Statement::CreateDateStatement { line, .. } => *line, Statement::CreateTimeStatement { line, .. } => *line, - Statement::CreateDateStatement { line, .. } => *line, - Statement::CreateTimeStatement { line, .. } => *line, }, column: match stmt { Statement::VariableDeclaration { column, .. } => *column, @@ -988,8 +986,6 @@ impl Analyzer { Statement::MapCreation { column, .. } => *column, Statement::CreateDateStatement { column, .. } => *column, Statement::CreateTimeStatement { column, .. } => *column, - Statement::CreateDateStatement { column, .. } => *column, - Statement::CreateTimeStatement { column, .. } => *column, }, }); then_nodes.push(then_node_idx); @@ -1056,10 +1052,6 @@ impl Analyzer { Statement::MapCreation { line, .. } => *line, Statement::CreateDateStatement { line, .. } => *line, Statement::CreateTimeStatement { line, .. } => *line, - Statement::CreateDateStatement { line, .. } => *line, - Statement::CreateTimeStatement { line, .. } => *line, - Statement::CreateDateStatement { line, .. } => *line, - Statement::CreateTimeStatement { line, .. } => *line, }, column: match stmt { Statement::VariableDeclaration { column, .. } => *column, @@ -1110,10 +1102,6 @@ impl Analyzer { Statement::MapCreation { column, .. } => *column, Statement::CreateDateStatement { column, .. } => *column, Statement::CreateTimeStatement { column, .. } => *column, - Statement::CreateDateStatement { column, .. } => *column, - Statement::CreateTimeStatement { column, .. } => *column, - Statement::CreateDateStatement { column, .. } => *column, - Statement::CreateTimeStatement { column, .. } => *column, }, }); else_nodes.push(else_node_idx); diff --git a/src/interpreter/mod.rs b/src/interpreter/mod.rs index 54869988..8ce1af0e 100644 --- a/src/interpreter/mod.rs +++ b/src/interpreter/mod.rs @@ -2127,7 +2127,7 @@ impl Interpreter { let today = chrono::Local::now().date_naive(); Value::Date(Rc::new(today)) }; - + env.borrow_mut().define(name, date_value); Ok((Value::Null, ControlFlow::None)) } @@ -2145,7 +2145,7 @@ impl Interpreter { let now = chrono::Local::now().time(); Value::Time(Rc::new(now)) }; - + env.borrow_mut().define(name, time_value); Ok((Value::Null, ControlFlow::None)) } @@ -2160,7 +2160,7 @@ impl Interpreter { // Get the list from the environment let list_val = env.borrow().get(list_name).ok_or_else(|| { - RuntimeError::new(format!("Undefined variable: {}", list_name), *line, *column) + RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) })?; match list_val { @@ -2204,7 +2204,7 @@ impl Interpreter { // Get the list from the environment let list_val = env.borrow().get(list_name).ok_or_else(|| { - RuntimeError::new(format!("Undefined variable: {}", list_name), *line, *column) + RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) })?; match list_val { @@ -2230,7 +2230,7 @@ impl Interpreter { } => { // Get the list from the environment let list_val = env.borrow().get(list_name).ok_or_else(|| { - RuntimeError::new(format!("Undefined variable: {}", list_name), *line, *column) + RuntimeError::new(format!("Undefined variable: {list_name}"), *line, *column) })?; match list_val { diff --git a/src/parser/mod.rs b/src/parser/mod.rs index 0fa6ed65..ee7dcf3c 100644 --- a/src/parser/mod.rs +++ b/src/parser/mod.rs @@ -1376,7 +1376,9 @@ impl<'a> Parser<'a> { /// /// # Examples /// - /// ``` + /// ```ignore + /// # use wfl::parser::Parser; + /// # use wfl::lexer::token::Token; /// let mut parser = Parser::new(&tokens); /// parser.expect_token(Token::Colon, "Expected ':' after name")?; /// ``` @@ -1410,7 +1412,7 @@ impl<'a> Parser<'a> { /// /// # Examples /// - /// ``` + /// ```ignore /// // Assuming the parser is positioned at a "divided" token: /// if parser.peek_divided_by() { /// // The next token is "by" @@ -1444,7 +1446,7 @@ impl<'a> Parser<'a> { /// /// # Examples /// - /// ``` + /// ```ignore /// // Parses an arithmetic expression with correct precedence /// let expr = parser.parse_binary_expression(0)?; /// // Example: "a plus b times c" parses as a + (b * c) @@ -2025,6 +2027,8 @@ impl<'a> Parser<'a> { } Token::Identifier(name) => { self.tokens.next(); + let token_line = token.line; + let token_column = token.column; // Check for property access (dot notation) if let Some(next_token) = self.tokens.peek().cloned() { @@ -2074,13 +2078,13 @@ impl<'a> Parser<'a> { return Ok(Expression::MethodCall { object: Box::new(Expression::Variable( name.clone(), - token.line, - token.column, + token_line, + token_column, )), method: property_name.clone(), arguments, - line: token.line, - column: token.column, + line: token_line, + column: token_column, }); } @@ -2088,12 +2092,12 @@ impl<'a> Parser<'a> { return Ok(Expression::PropertyAccess { object: Box::new(Expression::Variable( name.clone(), - token.line, - token.column, + token_line, + token_column, )), property: property_name.clone(), - line: token.line, - column: token.column, + line: token_line, + column: token_column, }); } else { return Err(ParseError::new( @@ -2105,8 +2109,8 @@ impl<'a> Parser<'a> { } else { return Err(ParseError::new( "Expected property name after '.'".to_string(), - token.line, - token.column, + token_line, + token_column, )); } } else if let Token::Identifier(id) = &next_token.token @@ -2116,8 +2120,6 @@ impl<'a> Parser<'a> { let arguments = self.parse_argument_list()?; - let token_line = token.line; - let token_column = token.column; return Ok(Expression::ActionCall { name: name.clone(), arguments, @@ -2129,9 +2131,6 @@ impl<'a> Parser<'a> { let is_standalone = false; - let token_line = token.line; - let token_column = token.column; - if is_standalone { exec_trace!( "Found standalone identifier '{}', treating as function call", @@ -2492,6 +2491,53 @@ impl<'a> Parser<'a> { if let Ok(mut expr) = result { while let Some(token) = self.tokens.peek().cloned() { match &token.token { + // Support direct index access: listName index (e.g., states 1) + Token::IntLiteral(index) => { + // Only treat as index access for specific base kinds and when on the same source line + if matches!( + expr, + Expression::Variable(_, _, _) + | Expression::IndexAccess { .. } + | Expression::FunctionCall { .. } + | Expression::PropertyAccess { .. } + | Expression::MethodCall { .. } + ) { + // Extract base expr span for anchoring + let (base_line, base_col) = match &expr { + Expression::Variable(_, line, col) + | Expression::IndexAccess { + line, column: col, .. + } + | Expression::FunctionCall { + line, column: col, .. + } + | Expression::PropertyAccess { + line, column: col, .. + } + | Expression::MethodCall { + line, column: col, .. + } => (*line, *col), + _ => (token.line, token.column), + }; + // Guard: require same line to avoid cross-line capture + if token.line != base_line { + break; + } + self.tokens.next(); // Consume the number + expr = Expression::IndexAccess { + collection: Box::new(expr), + index: Box::new(Expression::Literal( + Literal::Integer(*index), + token.line, + token.column, + )), + line: base_line, + column: base_col, + }; + } else { + break; // Not an index access; stop parsing postfix operators + } + } Token::KeywordOf => { self.tokens.next(); // Consume "of" @@ -5770,7 +5816,7 @@ impl<'a> Parser<'a> { /// /// # Examples /// - /// ``` + /// ```ignore /// // Parses: create date my_date /// // Parses: create date my_date as some_expression /// let stmt = parser.parse_create_date_statement()?; @@ -5812,7 +5858,7 @@ impl<'a> Parser<'a> { /// /// # Examples /// - /// ``` + /// ```ignore /// // Parses: create time start_time /// let stmt = parser.parse_create_time_statement().unwrap(); /// diff --git a/src/parser/tests.rs b/src/parser/tests.rs index 06964c48..ea3881e2 100644 --- a/src/parser/tests.rs +++ b/src/parser/tests.rs @@ -595,6 +595,7 @@ end pattern"#; /// Tests parsing of a pattern definition with alternative patterns. /// /// Verifies that the parser correctly recognizes a pattern definition containing an alternative between two string literals, ensuring the resulting AST represents an `Alternative` with the expected literals. +#[allow(dead_code)] fn test_parse_alternative_pattern() { let input = r#"create pattern greeting: "hello" or "hi" @@ -717,6 +718,7 @@ fn test_chained_binary_operations_parsing() { /// ``` /// debug_token_sequence(); // Prints the token sequence for inspection /// ``` +#[allow(dead_code)] fn debug_token_sequence() { let input = "store result as 1 plus 2 plus 3"; let tokens = lex_wfl_with_positions(input); diff --git a/syntax_test/pattern.wfl b/syntax_test/pattern.wfl index 5ad0f66b..209059ba 100644 --- a/syntax_test/pattern.wfl +++ b/syntax_test/pattern.wfl @@ -123,7 +123,25 @@ end map // create date and time create date today create time now -display "Date and time created successfully!" +display "Date and time created successfully!" + + +//test join command +create list states: + add "Azusa" + add "Nakano" + add "is" + add "cute" +end list + +store output as states 0 + " " + states 1 + " " + states 2 + " " + states 3 +display "Joined states: " + output + +display "states: " + states + + + + // Compile patterns for performance (future feature) // compile pattern "email" as email_validator diff --git a/syntax_test/pattern_debug.txt b/syntax_test/pattern_debug.txt index 81c08b48..9e270372 100644 --- a/syntax_test/pattern_debug.txt +++ b/syntax_test/pattern_debug.txt @@ -1,19 +1,19 @@ === WFL Debug Report === Script: syntax_test/pattern.wfl -Time: 2025-08-07 11:27:29 +Time: 2025-08-09 11:43:11 === Error Summary === -Runtime error at line 92, column 30: Variable 'count' can only be used inside count loops. Use 'count from X to Y:' to create a count loop. +Runtime error at line 137, column 68: Index 4 out of bounds for list of length 4 === Stack Trace === -In main script at line 92, column 30 +In main script at line 137, column 68 === Source Code === - 90: divide cn1 by 2 - 91: ->> 92: display "Final count is: " + count - 93: - 94: // Compile patterns for performance (future feature) + 135: end list + 136: +>> 137: store output as states 1 + " " + states 2 + " " + states 3 + " " + states 4 + 138: display "Joined states: " + output + 139: === Local Variables === (No local variables in global scope)