V0.4.1 (#51)

  - Added support for applying and simplifying rules using the HCRE library
  - Introduced new transformation modes `rule-apply` and `rule-simplify` 
  - Documentation updates
  - Changed URL processing to be kinder to hosts within reason to reduce potential "timeout" errors

README.md

@@ -24,6 +24,8 @@
 - **Analysis Tools:** Analyze input data with statistics and verbose output.
 - **Template Files:** Use template files to apply multiple transformations and
   operations to input data.
+- **Rule Application & Simplification:** Apply rules to input data and simplify
+  rules for optimization by using the [HCRE](https://git.launchpad.net/hcre/tree/README.md) library.
 
 ## Getting Started:
 
@@ -52,7 +54,7 @@ git clone https://github.com/JakeWnuk/ptt && cd ptt && docker build -t ptt . &&
 ---
 ### Usage:
 ```
-Usage of Password Transformation Tool (ptt) version (0.4.0):
+Usage of Password Transformation Tool (ptt) version (0.4.1):
 
 ptt [options] [...]
 Accepts standard input and/or additonal arguments.
@@ -70,7 +72,7 @@ These modify or filter the transformation mode.
   -i value
         Starting index for transformations if applicable. Accepts ranges separated by '-'.
   -ic
-        Ignore case when processing output and converts to lowercase.
+        Ignore case when processing output and converts all output to lowercase.
   -k value
         Only keep items in a file.
   -l value
@@ -138,6 +140,8 @@ These create or alter based on the selected mode.
         Transforms input by creating append rules.
   -t rule-append-remove
         Transforms input by creating append-remove rules.
+  -t rule-apply -tf [file]
+        Transforms input by applying rules to strings using the HCRE library.
   -t rule-insert -i [index]
         Transforms input by creating insert rules starting at index.
   -t rule-overwrite -i [index]
@@ -148,6 +152,8 @@ These create or alter based on the selected mode.
         Transforms input by creating prepend-remove rules.
   -t rule-prepend-toggle
         Transforms input by creating prepend-toggle rules.
+  -t rule-simplify
+        Transforms input by simplifying rules to efficient equivalents.
   -t rule-toggle -i [index]
         Transforms input by creating toggle rules starting at index.
   -t substring -i [index]

docs/SECURITY.md

@@ -1,9 +1,15 @@
 # Security Documentation for Password Transformation Tool (PTT)
 
 ## Notices
-- No additional notes.
+- As of `v0.4.1`, the HCRE library is included in the project. This library is
+  used to support complex rule features and has been locked to the following
+  versions:
+  - launchpad.net/hcre v0.0.0-20241130145909-c832018180b1 h1:lfPqGETHlSypBMeJtjMAFTthkaE/Wkxgu1vzYpwKdEI=
+  - launchpad.net/hcre v0.0.0-20241130145909-c832018180b1/go.mod h1:Dq78e8vypvdrOQt+VImkJcRq/6GHM1XGvO9/T1nr18M=
+
 ## Notes:
 - `pkg/utils/utils.go:9` use of `crypto/rand` over `math/rand` is not needed in this module.
+
 ## Last SAST Scan:
-- `11/08/2024` - `v0.4.0`
-- `10/16/2024` - `v0.3.7`
+- `v0.4.1`
+    - Included `hcre` library

docs/USAGE.md

@@ -1,5 +1,5 @@
 # Password Transformation Tool (PTT) Usage Guide
-> Version 0.4.0
+> Version 0.4.1
 ## Table of Contents
 - [Introduction](#introduction)
 - [Installation](#installation)
@@ -27,6 +27,8 @@
   - [Hex and Dehex](#hex-and-dehex)
   - [Substrings](#substrings)
   - [Regram](#regram)
+  - [Rule Application](#rule-application)
+  - [Rule Simplification](#rule-simplification)
 
 ## Introduction
 The Password Transformation Tool (PTT) is a command-line utility that allows users to transform passwords using various methods. This guide will provide instructions on how to install and use the tool.
@@ -110,7 +112,7 @@ There are some additional notes when importing data and getting started:
 - `ptt -n 50`: Show verbose statistics output with a maximum of 50 items.
 - `ptt -o [FILE]`: Show output and save JSON output to a file.
 - `ptt -md`: Show output as a Markdown table.
-- `ptt -ic`: Ignore case when creating output.
+- `ptt -ic`: Ignore case when creating output and convert to lowercase.
 - These options are available for all transformations.
 #### Rockyou Examples:
 `ptt -f rockyou.txt -t pop -l 4-5`:
@@ -499,4 +501,17 @@ ptt -f <input_file> -t regram -w <word_count>
 ```
 The `regram` transformation will generate new n-grams from the input by combining words from the input. The number of words to use in the n-gram is specified by the `-w` flag. The output will be the new n-grams generated from the input.
 
+### Rule Application
+This mode allows applying rules to the input. The syntax is as follows:
+```
+ptt -f <input_file> -t rule-apply -tf <rule_file>
+```
+The `rule-apply` transformation will apply rules from the rule file to the input. The rule file should contain the rules to be applied to the input. The output will be the input with the rules applied. This feature is enabled by the work done on the [HCRE](https://git.launchpad.net/hcre/tree/README.md) project. Please consider visiting the project for more information and supporting the author.
 
+### Rule Simplification
+This mode allows simplifying rules from the input. The syntax is as follows:
+```
+ptt -f <input_file> -t rule-simplify
+```
+The `rule-simplify` transformation will simplify rules from the input. The output will be the simplified rules equivalent to the input. This feature is enabled by the work done on the [HCRE](https://git.launchpad.net/hcre/tree/README.md) project. Please consider visiting the project for more information and supporting the author.
+```

go.mod

@@ -1,8 +1,12 @@
 module github.com/jakewnuk/ptt
 
-go 1.23.0
+go 1.23.1
+
+toolchain go1.23.2
 
 require (
 	golang.org/x/net v0.24.0
 	golang.org/x/text v0.14.0
 )
+
+require launchpad.net/hcre v0.0.0-20241130145909-c832018180b1 // indirect

go.sum

@@ -2,3 +2,5 @@ golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
 golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
 golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+launchpad.net/hcre v0.0.0-20241130145909-c832018180b1 h1:lfPqGETHlSypBMeJtjMAFTthkaE/Wkxgu1vzYpwKdEI=
+launchpad.net/hcre v0.0.0-20241130145909-c832018180b1/go.mod h1:Dq78e8vypvdrOQt+VImkJcRq/6GHM1XGvO9/T1nr18M=

main.go

@@ -15,7 +15,7 @@ import (
 	"github.com/jakewnuk/ptt/pkg/utils"
 )
 
-var version = "0.4.0"
+var version = "0.4.1"
 var wg sync.WaitGroup
 var mutex = &sync.Mutex{}
 var retain models.FileArgumentFlag
@@ -67,6 +67,8 @@ func main() {
 			"substring -i [index]":                  "Transforms input by extracting substrings starting at index and ending at index.",
 			"replace-all -tf [file]":                "Transforms input by replacing all strings with all matches from a ':' separated file.",
 			"regram -w [words]":                     "Transforms input by 'regramming' sentences into new n-grams with a given number of words.",
+			"rule-apply -tf [file]":                 "Transforms input by applying rules to strings using the HCRE library.",
+			"rule-simplify":                         "Transforms input by simplifying rules to efficient equivalents.",
 		}
 
 		// Sort and print transformation modes
@@ -96,7 +98,7 @@ func main() {
 	bypassMap := flag.Bool("b", false, "Bypass map creation and use stdout as primary output. Disables some options.")
 	debugMode := flag.Int("d", 0, "Enable debug mode with verbosity levels [0-2].")
 	URLParsingMode := flag.Int("p", 0, "Change parsing mode for URL input. [0 = Strict, 1 = Permissive, 2 = Maximum].")
-	ignoreCase := flag.Bool("ic", false, "Ignore case when processing output and converts to lowercase.")
+	ignoreCase := flag.Bool("ic", false, "Ignore case when processing output and converts all output to lowercase.")
 	flag.Var(&retain, "k", "Only keep items in a file.")
 	flag.Var(&remove, "r", "Only keep items not in a file.")
 	flag.Var(&readFiles, "f", "Read additional files for input.")

pkg/rule/rule.go

@@ -10,6 +10,7 @@ import (
 	"unicode"
 
 	"github.com/jakewnuk/ptt/pkg/utils"
+	"launchpad.net/hcre"
 )
 
 // ----------------------------------------------------------------------------
@@ -471,3 +472,79 @@ func ToggleRules(items map[string]int, index string, end string, bypass bool, de
 	}
 	return returnMap
 }
+
+// ApplyRulesHCRE uses the HCRE library to apply rules to a map of items
+// and returns the results
+//
+// Args:
+// items (map[string]int): Items to use in the operation
+// rules (map[string]int): Rules to use in the operation
+// bypass (bool): If true, the map is not used for output or filtering
+// debug (bool): If true, print additional debug information to stderr
+//
+// Returns:
+// returnMap (map[string]int): Map of items to return
+func ApplyRulesHCRE(items map[string]int, rules map[string]int, bypass bool, debug bool) (returnMap map[string]int) {
+	returnMap = make(map[string]int)
+	for key, value := range items {
+		for rule, _ := range rules {
+
+			rr, err := hcre.Compile(rule)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "[!] Error: %s\n", err)
+				os.Exit(1)
+			}
+			applyRule := rr.Simplify().Apply([]byte(key))
+
+			if debug {
+				fmt.Fprintf(os.Stderr, "[?] ApplyRulesHCRE:\n")
+				fmt.Fprintf(os.Stderr, "Key: %s\n", key)
+				fmt.Fprintf(os.Stderr, "Rule: %s\n", rule)
+				fmt.Fprintf(os.Stderr, "ApplyRule: %s\n", applyRule)
+			}
+
+			if applyRule != nil && !bypass {
+				returnMap[string(applyRule)] = value
+			} else if applyRule != nil && bypass {
+				fmt.Println(string(applyRule))
+			}
+		}
+	}
+	return returnMap
+}
+
+// SimplifyRules simplifies rules by simplifying rules to optimized equivalents
+// using the HCRE library
+//
+// Args:
+// items (map[string]int): Items to use in the operation
+// bypass (bool): If true, the map is not used for output or filtering
+// debug (bool): If true, print additional debug information to stderr
+//
+// Returns:
+// returnMap (map[string]int): Map of items to return
+func SimplifyRules(items map[string]int, bypass bool, debug bool) (returnMap map[string]int) {
+	returnMap = make(map[string]int)
+	for key, value := range items {
+
+		rr, err := hcre.Compile(key)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "[!] Error: %s\n", err)
+			os.Exit(1)
+		}
+		simplifyRule := rr.Simplify().String()
+
+		if debug {
+			fmt.Fprintf(os.Stderr, "[?] SimplifyRules:\n")
+			fmt.Fprintf(os.Stderr, "Key: %s\n", key)
+			fmt.Fprintf(os.Stderr, "SimplifyRule: %s\n", simplifyRule)
+		}
+
+		if simplifyRule != "" && !bypass {
+			returnMap[simplifyRule] = value
+		} else if simplifyRule != "" && bypass {
+			fmt.Println(simplifyRule)
+		}
+	}
+	return returnMap
+}

pkg/transform/transform.go

@@ -138,6 +138,16 @@ func TransformationController(input map[string]int, mode string, startingIndex i
 			os.Exit(1)
 		}
 		output = GenerateNGramMap(input, wordRangeStart, wordRangeEnd, bypass, functionDebug)
+	case "rule-apply", "apply":
+		fmt.Fprintf(os.Stderr, "[*] This transformation mode expects a rule file to apply.\n")
+		if len(transformationFilesMap) == 0 {
+			fmt.Fprintf(os.Stderr, "[!] Apply operations require use of one or more -tf flags to specify one or more files.\n")
+			os.Exit(1)
+		}
+		output = rule.ApplyRulesHCRE(input, transformationFilesMap, bypass, functionDebug)
+	case "rule-simplify", "simplify":
+		fmt.Fprintf(os.Stderr, "[*] This transformation mode expects rule input to simplify.\n")
+		output = rule.SimplifyRules(input, bypass, functionDebug)
 	default:
 		output = input
 	}

pkg/utils/utils.go

@@ -51,7 +51,7 @@ func TrackLoadTime(done <-chan bool, work string) {
 		case t := <-ticker.C:
 			elapsed := t.Sub(start)
 			memUsage := GetMemoryUsage()
-			fmt.Fprintf(os.Stderr, "[-] Please wait. Elapsed: %02d:%02d:%02d.%03d. Memory Usage: %.2f MB.\n", int(t.Sub(start).Hours()), int(t.Sub(start).Minutes())%60, int(t.Sub(start).Seconds())%60, elapsed.Milliseconds()%1000, memUsage)
+			fmt.Fprintf(os.Stderr, "[-] Please wait loading. Elapsed: %02d:%02d:%02d.%03d. Memory Usage: %.2f MB.\n", int(t.Sub(start).Hours()), int(t.Sub(start).Minutes())%60, int(t.Sub(start).Seconds())%60, elapsed.Milliseconds()%1000, memUsage)
 		}
 	}
 }
@@ -213,23 +213,9 @@ func ReadURLsToMap(urls []string, parsingMode int, debugMode int) (map[string]in
 		}
 	}()
 
-	prevURL := ""
-	sleepOnStart := false
+	sleepOnStart := true
 	for _, iURL := range urls {
 		if IsValidURL(iURL) {
-
-			parsedURL, err := url.Parse(iURL)
-			if err != nil {
-				fmt.Println("[!] Error parsing URL:", err)
-				continue
-			}
-			if parsedURL.Host == prevURL {
-				sleepOnStart = true
-			} else {
-				sleepOnStart = false
-			}
-			prevURL = parsedURL.Host
-
 			wg.Add(1)
 			go ProcessURL(iURL, ch, &wg, parsingMode, debugMode, sleepOnStart)
 
@@ -306,9 +292,8 @@ func CombineMaps(maps ...map[string]int) map[string]int {
 //	None
 func ProcessURL(url string, ch chan<- string, wg *sync.WaitGroup, parsingMode int, debugMode int, sleepOnStart bool) {
 	defer wg.Done()
-
 	var resp *http.Response
-	throttleInterval := 30
+	throttleInterval := 90
 	source := rand.NewSource(time.Now().UnixNano())
 	r := rand.New(source)
 	const maxRetries = 3
@@ -328,7 +313,7 @@ func ProcessURL(url string, ch chan<- string, wg *sync.WaitGroup, parsingMode in
 	}
 
 	if sleepOnStart {
-		time.Sleep(time.Second * time.Duration(throttleInterval) * time.Duration(r.Intn(10)))
+		time.Sleep(time.Second * time.Duration(r.Intn(throttleInterval)))
 	}
 
 	for attempts := 0; attempts <= maxRetries; attempts++ {
@@ -359,15 +344,11 @@ func ProcessURL(url string, ch chan<- string, wg *sync.WaitGroup, parsingMode in
 
 		// Check the response code for throttling
 		if resp.StatusCode == http.StatusTooManyRequests {
-			fmt.Fprintf(os.Stderr, "[!] Throttling detected. Waiting %d seconds before retrying.\n", throttleInterval)
-			time.Sleep(time.Second * time.Duration(throttleInterval))
-			throttleInterval++
-		}
-
-		fmt.Fprintf(os.Stderr, "[+] Requested %s. Attempt [%d/%d]. Response Code: %s. Content-Type: %s. \n", url, attempts, maxRetries, resp.Status, resp.Header.Get("Content-Type"))
-		if resp.StatusCode == http.StatusTooManyRequests {
-			time.Sleep(time.Second * time.Duration(throttleInterval) * time.Duration(r.Intn(10)))
-			continue
+			throttleInterval += 30
+			time.Sleep(time.Second * time.Duration(throttleInterval*(r.Intn(3)+1)))
+			fmt.Fprintf(os.Stderr, "[!] Requested %s. Attempt [%d/%d]. Response Code: %s. Waiting %d seconds before retrying. \n", url, attempts, maxRetries, resp.Status, throttleInterval)
+		} else {
+			fmt.Fprintf(os.Stderr, "[+] Requested %s. Attempt [%d/%d]. Response Code: %s. Content-Type: %s. \n", url, attempts, maxRetries, resp.Status, resp.Header.Get("Content-Type"))
 		}
 
 		if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusNotFound {
@@ -691,7 +672,6 @@ func ReadJSONToArray(fs models.FileSystem, filenames []string) []models.Template
 func ProcessURLFile(filePath string, ch chan<- string, wg *sync.WaitGroup, parsingMode int, debugMode int) {
 	defer wg.Done()
 	sleepOnStart := false
-	prevURL := ""
 
 	file, err := os.Open(filePath)
 	if err != nil {
@@ -704,19 +684,6 @@ func ProcessURLFile(filePath string, ch chan<- string, wg *sync.WaitGroup, parsi
 	for scanner.Scan() {
 		line := scanner.Text()
 		if IsValidURL(line) {
-
-			parsedURL, err := url.Parse(line)
-			if err != nil {
-				fmt.Println("[!] Error parsing URL:", err)
-				continue
-			}
-			if parsedURL.Host == prevURL {
-				sleepOnStart = true
-			} else {
-				sleepOnStart = false
-			}
-			prevURL = parsedURL.Host
-
 			wg.Add(1)
 			go ProcessURL(line, ch, wg, parsingMode, debugMode, sleepOnStart)
 		} else {