v0.3.2 (#35)

- swap is now `swap-single` - replace is now `replace-all` - aliased both for prior support - added more verbosity messages for flags like -d -k -r -l -m -o - `-n` will now affect the total amount of output shown, not just in verbose modes - update docs - bug fix with `match` and `-b` where logic was forgotten
JakeWnuk · Sep 12, 2024 · 649811a · 649811a
1 parent eb20e27
commit 649811a
Show file tree

Hide file tree

Showing 7 changed files with 160 additions and 42 deletions.
diff --git a/README.md b/README.md
@@ -40,7 +40,7 @@ git clone https://github.com/JakeWnuk/ptt && cd ptt && docker build -t ptt . &&
 
 ### Usage:
 ```
-Usage of Password Transformation Tool (ptt) version (0.3.1):
+Usage of Password Transformation Tool (ptt) version (0.3.2):
 
 ptt [options] [...]
 Accepts standard input and/or additonal arguments.
@@ -64,7 +64,7 @@ These modify or filter the transformation mode.
   -m int
         Minimum numerical frequency to include in output.
   -n int
-        Maximum number of items to display in verbose statistics output. (default 25)
+        Maximum number of items to return in output. 
   -o string
         Output to JSON file in addition to stdout.
   -p int
@@ -114,7 +114,7 @@ These create or alter based on the selected mode.
         Transforms input by swapping tokens from a partial mask file and a input file.
   -t passphrase -w [words] -tf [file]
         Transforms input by randomly generating passphrases with a given number of words and separators from a file.
-  -t replace -tf [file]
+  -t replace-all -tf [file]
         Transforms input by replacing all strings with all matches from a ':' separated file.
   -t rule-append
         Transforms input into append rules.
@@ -134,7 +134,7 @@ These create or alter based on the selected mode.
         Transforms input into toggle rules starting at index.
   -t substring -i [index]
         Transforms input by extracting substrings starting at index and ending at index.
-  -t swap -tf [file]
-        Transforms input by swapping tokens with exact matches from a ':' separated file.
+  -t swap-single -tf [file]
+        Transforms input by swapping tokens once per string per replacement with exact matches from a ':' separated file.
 -------------------------------------------------------------------------------------------------------------
 ```
diff --git a/docs/USAGE.md b/docs/USAGE.md
@@ -119,11 +119,11 @@ These flags work with files and directories.
 - `-k`: Only keep items in a file.
 - `-l`: Only output items of a certain length (does not adjust for rules). Accepts ranges separated by '-'.
 - `-m`: Minimum numerical frequency to include in output.
-- `-n`: Maximum number of items to display in verbose statistics output. (default 25)
+- `-n`: Maximum number of items to return in output.
 - `-o`: Output to JSON file in addition to stdout.
 - `-p`: Change parsing mode for URL input. [0 = Strict, 1 = Permissive, 2 = Maximum].
 - `-r`: Only keep items not in a file.
-- `-rm`: Replacement mask for transformations if applicable. (default "uldsb")
+- `-rm`: Replacement mask for transformations if applicable. (default "uldsbt")
 - `-t`: Transformation to apply to input.
 - `-tf`: Read additional files for transformations if applicable.
 - `-tp`: Read a template file for multiple transformations and operations.
@@ -134,27 +134,52 @@ These flags work with files and directories.
 
 #### Transformations:
 The following transformations can be used with the `-t` flag:
-- `rule-append`: Transforms input into append rules.
-- `rule-append-remove`: Transforms input into append-remove rules
-- `rule-prepend`: Transforms input into prepend rules.
-- `rule-prepend-remove`: Transforms input into prepend-remove rules.
-- `rule-prepend-toggle`: Transforms input into prepend-toggle rules.
-- `rule-insert`: Transforms input into insert rules starting at index.
-- `rule-overwrite`: Transforms input into overwrite rules starting at index.
-- `rule-toggle`: Transforms input into toggle rules starting at index.
-- `encode`: Transforms input by HTML and Unicode escape encoding.
-- `decode`: Transforms input by HTML and Unicode escape decoding.
-- `hex`: Transforms input by encoding strings into $HEX[...] format.
-- `dehex`: Transforms input by decoding $HEX[...] formatted
-- `mask`: Transforms input by masking characters with provided mask.
-- `mask-remove`: Transforms input by removing characters with provided mask characters.
-- `substring`: Transforms input by extracting substrings starting at index and ending at index.
-- `mask-retain`: Transforms input by creating masks that still retain strings from file.
-- `mask-match`: Transforms input by keeping only strings with matching masks from a mask file
-- `mask-swap`: Transforms input by swapping tokens with exact matches from a ':' separated file.
-- `mask-pop`: Transforms input by generating tokens from popping strings at character boundaries.
-- `mask-swap`: Transforms input by swapping tokens from a partial mask file and a input file.
-- `passphrase`: Transforms input by randomly generating passphrases with a given number of words and separators from a file.
+```
+  -t decode
+        Transforms input by HTML and Unicode escape decoding.
+  -t dehex
+        Transforms input by decoding $HEX[...] formatted strings.
+  -t encode
+        Transforms input by HTML and Unicode escape encoding.
+  -t hex
+        Transforms input by encoding strings into $HEX[...] format.
+  -t mask -rm [uldsb] -v
+        Transforms input by masking characters with provided mask.
+  -t mask-match -tf [file]
+        Transforms input by keeping only strings with matching masks from a mask file.
+  -t mask-pop -rm [uldsbt]
+        Transforms input by generating tokens from popping strings at character boundaries.
+  -t mask-remove -rm [uldsb]
+        Transforms input by removing characters with provided mask characters.
+  -t mask-retain -rm [uldsb] -tf [file]
+        Transforms input by creating masks that still retain strings from file.
+  -t mask-swap -tf [file]
+        Transforms input by swapping tokens from a partial mask file and a input file.
+  -t passphrase -w [words] -tf [file]
+        Transforms input by randomly generating passphrases with a given number of words and separators from a file.
+  -t replace-all -tf [file]
+        Transforms input by replacing all strings with all matches from a ':' separated file.
+  -t rule-append
+        Transforms input into append rules.
+  -t rule-append-remove
+        Transforms input into append-remove rules.
+  -t rule-insert -i [index]
+        Transforms input into insert rules starting at index.
+  -t rule-overwrite -i [index]
+        Transforms input into overwrite rules starting at index.
+  -t rule-prepend
+        Transforms input into prepend rules.
+  -t rule-prepend-remove
+        Transforms input into prepend-remove rules.
+  -t rule-prepend-toggle
+        Transforms input into prepend-toggle rules. Creating camelCase and PascalCase.
+  -t rule-toggle -i [index]
+        Transforms input into toggle rules starting at index.
+  -t substring -i [index]
+        Transforms input by extracting substrings starting at index and ending at index.
+  -t swap-single -tf [file]
+        Transforms input by swapping tokens once per string per replacement with exact matches from a ':' separated file.
+```
 
 ### Examples
 
@@ -469,7 +494,7 @@ This document describes the ways to use PTT to create password cracking
 wordlists. There are several ways to generate wordlists using PTT:
 
 - `Direct Swapping`: Swapping characters directly with a `:` separated file.
-   This is implemented in the `swap` module.
+   This is implemented in the `swap-single` module.
 - `Replacing Text and Characters`: Replacing text and characters in a string.
   This is implemented in the `replace` module
 - `Token Popping`: Generates tokens by popping strings at character boundaries.
@@ -486,19 +511,19 @@ transformation can be used at a time.
 > Ensure input is provided in the correct format and does not contain hidden characters. `Dos2Unix` can be used to convert the file to proper format if needed.
 
 ### Direct Swapping
-The `swap` module swaps characters directly with a `:` separated file. The
+The `swap-single` module swaps characters directly with a `:` separated file. The
 syntax is as follows:
 ```
-ptt -f <input-file> -t swap -tf <replacement-file>
+ptt -f <input-file> -t swap-single -tf <replacement-file>
 ```
 The replacement file should contain the strings to be transformed as `PRIOR:POST`
 pairs. The replacements will be applied to the all instance in each line but
 only one swap is applied at once. This mode is ideal for substituting words or characters in a string.
 
 ### Replacing Text and Characters
-The `replace` module replaces text and characters in a string. This mode replaces all strings with all matches from a ':' separated file. The syntax is as follows:
+The `replace-all` module replaces text and characters in a string. This mode replaces all strings with all matches from a ':' separated file. The syntax is as follows:
 ```
-ptt -f <input-file> -t replace -tf <replacement-file>
+ptt -f <input-file> -t replace-all -tf <replacement-file>
 ```
 The replacement file should contain the strings to be transformed as
 `PRIOR:POST` pairs. The replacements will be applied to all instances in each

diff --git a/main.go b/main.go
@@ -15,7 +15,7 @@ import (
 	"github.com/jakewnuk/ptt/pkg/utils"
 )
 
-var version = "0.3.1"
+var version = "0.3.2"
 var wg sync.WaitGroup
 var mutex = &sync.Mutex{}
 var retain models.FileArgumentFlag
@@ -60,11 +60,11 @@ func main() {
 			"mask-retain -rm [uldsb] -tf [file]": "Transforms input by creating masks that still retain strings from file.",
 			"mask-pop -rm [uldsbt]":              "Transforms input by generating tokens from popping strings at character boundaries.",
 			"mask-match -tf [file]":              "Transforms input by keeping only strings with matching masks from a mask file.",
-			"swap -tf [file]":                    "Transforms input by swapping tokens with exact matches from a ':' separated file.",
+			"swap-single -tf [file]":             "Transforms input by swapping tokens once per string per replacement with exact matches from a ':' separated file.",
 			"mask-swap -tf [file]":               "Transforms input by swapping tokens from a partial mask file and a input file.",
 			"passphrase -w [words] -tf [file]":   "Transforms input by randomly generating passphrases with a given number of words and separators from a file.",
 			"substring -i [index]":               "Transforms input by extracting substrings starting at index and ending at index.",
-			"replace -tf [file]":                 "Transforms input by replacing all strings with all matches from a ':' separated file.",
+			"replace-all -tf [file]":             "Transforms input by replacing all strings with all matches from a ':' separated file.",
 		}
 
 		// Sort and print transformation modes
@@ -86,7 +86,7 @@ func main() {
 	verbose2 := flag.Bool("vv", false, "Show statistics output when possible.")
 	verbose3 := flag.Bool("vvv", false, "Show verbose statistics output when possible.")
 	minimum := flag.Int("m", 0, "Minimum numerical frequency to include in output.")
-	verboseStatsMax := flag.Int("n", 25, "Maximum number of items to display in verbose statistics output.")
+	outputVerboseMax := flag.Int("n", 0, "Maximum number of items to return in output.")
 	transformation := flag.String("t", "", "Transformation to apply to input.")
 	replacementMask := flag.String("rm", "uldsbt", "Replacement mask for transformations if applicable.")
 	jsonOutput := flag.String("o", "", "Output to JSON file in addition to stdout.")
@@ -109,6 +109,11 @@ func main() {
 		fmt.Fprintf(os.Stderr, "[*] Bypassing map creation and using stdout as primary output. Options are disabled.\n")
 	}
 
+	// Print debug information if requested
+	if *debugMode > 0 {
+		fmt.Fprintf(os.Stderr, "[*] Debug mode enabled with verbosity level %d.\n", *debugMode)
+	}
+
 	// Parse any retain, remove, or transformation file arguments
 	fs := &models.RealFileSystem{}
 	retainMap := utils.ReadFilesToMap(fs, retain)
@@ -169,16 +174,31 @@ func main() {
 		return
 	}
 
+	// Print remove frequency if provided
+	if *minimum > 0 {
+		fmt.Fprintf(os.Stderr, "[*] Removing items with frequency less than %d.\n", *minimum)
+	}
+
 	// Remove items under minimum frequency if provided
 	if *minimum > 0 {
 		primaryMap = format.RemoveMinimumFrequency(primaryMap, *minimum)
 	}
 
+	// Print length range if provided
+	if lenRange.Start > 0 || lenRange.End > 0 {
+		fmt.Fprintf(os.Stderr, "[*] Only outputting items between %d and %d characters.\n", lenRange.Start, lenRange.End)
+	}
+
 	// Remove items outside of length range if provided
 	if lenRange.Start > 0 || lenRange.End > 0 {
 		primaryMap = format.RemoveLengthRange(primaryMap, lenRange.Start, lenRange.End)
 	}
 
+	// Print retained and removed items if provided
+	if len(retainMap) > 0 || len(removeMap) > 0 {
+		fmt.Fprintf(os.Stderr, "[*] Retain/remove flags provided. Retaining %d and removing %d items.\n", len(retainMap), len(removeMap))
+	}
+
 	// Process retain and remove maps if provided
 	if len(retainMap) > 0 || len(removeMap) > 0 {
 		primaryMap, err = format.RetainRemove(primaryMap, retainMap, removeMap, *debugMode)
@@ -188,15 +208,25 @@ func main() {
 		}
 	}
 
+	// if -n is providied, filter ALL results to only that top amount
+	if *outputVerboseMax > 0 {
+		primaryMap = format.FilterTopN(primaryMap, *outputVerboseMax)
+	}
+
 	// Print output to stdout
 	if *verbose3 {
-		format.PrintStatsToSTDOUT(primaryMap, *verbose3, *verboseStatsMax)
+		format.PrintStatsToSTDOUT(primaryMap, *verbose3, *outputVerboseMax)
 	} else if *verbose2 {
-		format.PrintStatsToSTDOUT(primaryMap, *verbose3, *verboseStatsMax)
+		format.PrintStatsToSTDOUT(primaryMap, *verbose3, *outputVerboseMax)
 	} else {
 		format.PrintArrayToSTDOUT(primaryMap, *verbose)
 	}
 
+	// Print output location if provided
+	if *jsonOutput != "" {
+		fmt.Fprintf(os.Stderr, "[*] Saving output to JSON file: %s\n", *jsonOutput)
+	}
+
 	// Save output to JSON if provided
 	if *jsonOutput != "" {
 		err = format.SaveArrayToJSON(*jsonOutput, primaryMap)

diff --git a/pkg/format/format.go b/pkg/format/format.go
@@ -77,6 +77,10 @@ func PrintStatsToSTDOUT(freq map[string]int, verbose bool, max int) {
 	sort.Sort(sort.Reverse(p))
 	sort.Sort(sort.Reverse(normalizedP))
 
+	if max == 0 {
+		max = 25
+	}
+
 	if max > len(p) {
 		max = len(p)
 	}
@@ -528,6 +532,33 @@ func RemoveLengthRange(freq map[string]int, start int, end int) map[string]int {
 	return newFreq
 }
 
+// FilterTopN removes all but the top N items from a map of item frequencies
+// and returns a new map
+//
+// Args:
+// freq (map[string]int): A map of item frequencies
+// n (int): The number of items to retain
+//
+// Returns:
+// map[string]int: A new map of the top N item frequencies
+func FilterTopN(freq map[string]int, n int) map[string]int {
+	newFreq := make(map[string]int)
+	p := make(models.PairList, len(freq))
+	i := 0
+	for k, v := range freq {
+		p[i] = models.Pair{k, v}
+		i++
+	}
+	sort.Sort(sort.Reverse(p))
+	if n > len(p) {
+		n = len(p)
+	}
+	for i := 0; i < n; i++ {
+		newFreq[p[i].Key] = p[i].Value
+	}
+	return newFreq
+}
+
 // ----------------------------------------------------------------------------
 // Encoding Functions
 // ----------------------------------------------------------------------------

diff --git a/pkg/format/format_test.go b/pkg/format/format_test.go
@@ -13,6 +13,7 @@ import (
 // - RetainRemove()
 // - RemoveMinimumFrequency()
 // - RemoveLengthRange()
+// - FilterTopN()
 //
 // ** Encoding Functions **
 // - EncodeInputMap()
@@ -153,6 +154,35 @@ func TestRemoveLengthRange(t *testing.T) {
 	}
 }
 
+// Unit Test for FilterTopN()
+func TestFilterTopN(t *testing.T) {
+
+	// Define a test case struct
+	type testCase struct {
+		input  map[string]int
+		top    int
+		output map[string]int
+	}
+
+	type testCases []testCase
+
+	// Define a test case
+	tests := testCases{
+		{map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}, 2, map[string]int{"c": 3, "d": 4}},
+		{map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}, 3, map[string]int{"b": 2, "c": 3, "d": 4}},
+		{map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}, 4, map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}},
+		{map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}, 5, map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}},
+	}
+
+	// Run test cases
+	for _, test := range tests {
+		result := FilterTopN(test.input, test.top)
+		if utils.CheckAreMapsEqual(result, test.output) == false {
+			t.Errorf("FilterTopN() failed - expected: %v, got: %v", test.output, result)
+		}
+	}
+}
+
 // Unit Test for EncodeInputMap()
 func TestEncodeInputMap(t *testing.T) {
 

diff --git a/pkg/mask/mask.go b/pkg/mask/mask.go
@@ -352,7 +352,9 @@ func MakeMatchedMaskedMap(input map[string]int, replacementMask string, maskMap
 				}
 			}
 		case true:
-			fmt.Println(newKey)
+			if _, exists := maskMap[newKey]; exists {
+				fmt.Println(key)
+			}
 		}
 	}
 	return maskedMap

diff --git a/pkg/transform/transform.go b/pkg/transform/transform.go
@@ -99,7 +99,7 @@ func TransformationController(input map[string]int, mode string, startingIndex i
 			os.Exit(1)
 		}
 		output = mask.MakeMatchedMaskedMap(input, replacementMask, transformationFilesMap, bypass, functionDebug)
-	case "swap":
+	case "swap", "swap-single":
 		if len(transformationFilesMap) == 0 {
 			fmt.Fprintf(os.Stderr, "[!] Swap operations require use of one or more -tf flags to specify one or more files\n")
 			fmt.Fprintf(os.Stderr, "[!] This transformation mode requires a ':' separated list of keys to swap\n")
@@ -123,7 +123,7 @@ func TransformationController(input map[string]int, mode string, startingIndex i
 		output = MakePassphraseMap(input, transformationFilesMap, bypass, functionDebug, passphraseWords)
 	case "substring":
 		output = utils.SubstringMap(input, startingIndex, endingIndex, bypass, functionDebug)
-	case "replace":
+	case "replace-all", "replace":
 		if len(transformationFilesMap) == 0 {
 			fmt.Fprintf(os.Stderr, "[!] Replace operations require use of one or more -tf flags to specify one or more files\n")
 			os.Exit(1)