diff --git a/README.md b/README.md index 7c65dcb..393803b 100644 --- a/README.md +++ b/README.md @@ -40,7 +40,7 @@ git clone https://github.com/JakeWnuk/ptt && cd ptt && docker build -t ptt . && ### Usage: ``` -Usage of Password Transformation Tool (ptt) version (0.3.1): +Usage of Password Transformation Tool (ptt) version (0.3.2): ptt [options] [...] Accepts standard input and/or additonal arguments. @@ -64,7 +64,7 @@ These modify or filter the transformation mode. -m int Minimum numerical frequency to include in output. -n int - Maximum number of items to display in verbose statistics output. (default 25) + Maximum number of items to return in output. -o string Output to JSON file in addition to stdout. -p int @@ -114,7 +114,7 @@ These create or alter based on the selected mode. Transforms input by swapping tokens from a partial mask file and a input file. -t passphrase -w [words] -tf [file] Transforms input by randomly generating passphrases with a given number of words and separators from a file. - -t replace -tf [file] + -t replace-all -tf [file] Transforms input by replacing all strings with all matches from a ':' separated file. -t rule-append Transforms input into append rules. @@ -134,7 +134,7 @@ These create or alter based on the selected mode. Transforms input into toggle rules starting at index. -t substring -i [index] Transforms input by extracting substrings starting at index and ending at index. - -t swap -tf [file] - Transforms input by swapping tokens with exact matches from a ':' separated file. + -t swap-single -tf [file] + Transforms input by swapping tokens once per string per replacement with exact matches from a ':' separated file. ------------------------------------------------------------------------------------------------------------- ``` diff --git a/docs/USAGE.md b/docs/USAGE.md index 16428e1..e59a0c7 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -119,11 +119,11 @@ These flags work with files and directories. - `-k`: Only keep items in a file. - `-l`: Only output items of a certain length (does not adjust for rules). Accepts ranges separated by '-'. - `-m`: Minimum numerical frequency to include in output. -- `-n`: Maximum number of items to display in verbose statistics output. (default 25) +- `-n`: Maximum number of items to return in output. - `-o`: Output to JSON file in addition to stdout. - `-p`: Change parsing mode for URL input. [0 = Strict, 1 = Permissive, 2 = Maximum]. - `-r`: Only keep items not in a file. -- `-rm`: Replacement mask for transformations if applicable. (default "uldsb") +- `-rm`: Replacement mask for transformations if applicable. (default "uldsbt") - `-t`: Transformation to apply to input. - `-tf`: Read additional files for transformations if applicable. - `-tp`: Read a template file for multiple transformations and operations. @@ -134,27 +134,52 @@ These flags work with files and directories. #### Transformations: The following transformations can be used with the `-t` flag: -- `rule-append`: Transforms input into append rules. -- `rule-append-remove`: Transforms input into append-remove rules -- `rule-prepend`: Transforms input into prepend rules. -- `rule-prepend-remove`: Transforms input into prepend-remove rules. -- `rule-prepend-toggle`: Transforms input into prepend-toggle rules. -- `rule-insert`: Transforms input into insert rules starting at index. -- `rule-overwrite`: Transforms input into overwrite rules starting at index. -- `rule-toggle`: Transforms input into toggle rules starting at index. -- `encode`: Transforms input by HTML and Unicode escape encoding. -- `decode`: Transforms input by HTML and Unicode escape decoding. -- `hex`: Transforms input by encoding strings into $HEX[...] format. -- `dehex`: Transforms input by decoding $HEX[...] formatted -- `mask`: Transforms input by masking characters with provided mask. -- `mask-remove`: Transforms input by removing characters with provided mask characters. -- `substring`: Transforms input by extracting substrings starting at index and ending at index. -- `mask-retain`: Transforms input by creating masks that still retain strings from file. -- `mask-match`: Transforms input by keeping only strings with matching masks from a mask file -- `mask-swap`: Transforms input by swapping tokens with exact matches from a ':' separated file. -- `mask-pop`: Transforms input by generating tokens from popping strings at character boundaries. -- `mask-swap`: Transforms input by swapping tokens from a partial mask file and a input file. -- `passphrase`: Transforms input by randomly generating passphrases with a given number of words and separators from a file. +``` + -t decode + Transforms input by HTML and Unicode escape decoding. + -t dehex + Transforms input by decoding $HEX[...] formatted strings. + -t encode + Transforms input by HTML and Unicode escape encoding. + -t hex + Transforms input by encoding strings into $HEX[...] format. + -t mask -rm [uldsb] -v + Transforms input by masking characters with provided mask. + -t mask-match -tf [file] + Transforms input by keeping only strings with matching masks from a mask file. + -t mask-pop -rm [uldsbt] + Transforms input by generating tokens from popping strings at character boundaries. + -t mask-remove -rm [uldsb] + Transforms input by removing characters with provided mask characters. + -t mask-retain -rm [uldsb] -tf [file] + Transforms input by creating masks that still retain strings from file. + -t mask-swap -tf [file] + Transforms input by swapping tokens from a partial mask file and a input file. + -t passphrase -w [words] -tf [file] + Transforms input by randomly generating passphrases with a given number of words and separators from a file. + -t replace-all -tf [file] + Transforms input by replacing all strings with all matches from a ':' separated file. + -t rule-append + Transforms input into append rules. + -t rule-append-remove + Transforms input into append-remove rules. + -t rule-insert -i [index] + Transforms input into insert rules starting at index. + -t rule-overwrite -i [index] + Transforms input into overwrite rules starting at index. + -t rule-prepend + Transforms input into prepend rules. + -t rule-prepend-remove + Transforms input into prepend-remove rules. + -t rule-prepend-toggle + Transforms input into prepend-toggle rules. Creating camelCase and PascalCase. + -t rule-toggle -i [index] + Transforms input into toggle rules starting at index. + -t substring -i [index] + Transforms input by extracting substrings starting at index and ending at index. + -t swap-single -tf [file] + Transforms input by swapping tokens once per string per replacement with exact matches from a ':' separated file. +``` ### Examples @@ -469,7 +494,7 @@ This document describes the ways to use PTT to create password cracking wordlists. There are several ways to generate wordlists using PTT: - `Direct Swapping`: Swapping characters directly with a `:` separated file. - This is implemented in the `swap` module. + This is implemented in the `swap-single` module. - `Replacing Text and Characters`: Replacing text and characters in a string. This is implemented in the `replace` module - `Token Popping`: Generates tokens by popping strings at character boundaries. @@ -486,19 +511,19 @@ transformation can be used at a time. > Ensure input is provided in the correct format and does not contain hidden characters. `Dos2Unix` can be used to convert the file to proper format if needed. ### Direct Swapping -The `swap` module swaps characters directly with a `:` separated file. The +The `swap-single` module swaps characters directly with a `:` separated file. The syntax is as follows: ``` -ptt -f -t swap -tf +ptt -f -t swap-single -tf ``` The replacement file should contain the strings to be transformed as `PRIOR:POST` pairs. The replacements will be applied to the all instance in each line but only one swap is applied at once. This mode is ideal for substituting words or characters in a string. ### Replacing Text and Characters -The `replace` module replaces text and characters in a string. This mode replaces all strings with all matches from a ':' separated file. The syntax is as follows: +The `replace-all` module replaces text and characters in a string. This mode replaces all strings with all matches from a ':' separated file. The syntax is as follows: ``` -ptt -f -t replace -tf +ptt -f -t replace-all -tf ``` The replacement file should contain the strings to be transformed as `PRIOR:POST` pairs. The replacements will be applied to all instances in each diff --git a/main.go b/main.go index 60b37b1..64ea491 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,7 @@ import ( "github.com/jakewnuk/ptt/pkg/utils" ) -var version = "0.3.1" +var version = "0.3.2" var wg sync.WaitGroup var mutex = &sync.Mutex{} var retain models.FileArgumentFlag @@ -60,11 +60,11 @@ func main() { "mask-retain -rm [uldsb] -tf [file]": "Transforms input by creating masks that still retain strings from file.", "mask-pop -rm [uldsbt]": "Transforms input by generating tokens from popping strings at character boundaries.", "mask-match -tf [file]": "Transforms input by keeping only strings with matching masks from a mask file.", - "swap -tf [file]": "Transforms input by swapping tokens with exact matches from a ':' separated file.", + "swap-single -tf [file]": "Transforms input by swapping tokens once per string per replacement with exact matches from a ':' separated file.", "mask-swap -tf [file]": "Transforms input by swapping tokens from a partial mask file and a input file.", "passphrase -w [words] -tf [file]": "Transforms input by randomly generating passphrases with a given number of words and separators from a file.", "substring -i [index]": "Transforms input by extracting substrings starting at index and ending at index.", - "replace -tf [file]": "Transforms input by replacing all strings with all matches from a ':' separated file.", + "replace-all -tf [file]": "Transforms input by replacing all strings with all matches from a ':' separated file.", } // Sort and print transformation modes @@ -86,7 +86,7 @@ func main() { verbose2 := flag.Bool("vv", false, "Show statistics output when possible.") verbose3 := flag.Bool("vvv", false, "Show verbose statistics output when possible.") minimum := flag.Int("m", 0, "Minimum numerical frequency to include in output.") - verboseStatsMax := flag.Int("n", 25, "Maximum number of items to display in verbose statistics output.") + outputVerboseMax := flag.Int("n", 0, "Maximum number of items to return in output.") transformation := flag.String("t", "", "Transformation to apply to input.") replacementMask := flag.String("rm", "uldsbt", "Replacement mask for transformations if applicable.") jsonOutput := flag.String("o", "", "Output to JSON file in addition to stdout.") @@ -109,6 +109,11 @@ func main() { fmt.Fprintf(os.Stderr, "[*] Bypassing map creation and using stdout as primary output. Options are disabled.\n") } + // Print debug information if requested + if *debugMode > 0 { + fmt.Fprintf(os.Stderr, "[*] Debug mode enabled with verbosity level %d.\n", *debugMode) + } + // Parse any retain, remove, or transformation file arguments fs := &models.RealFileSystem{} retainMap := utils.ReadFilesToMap(fs, retain) @@ -169,16 +174,31 @@ func main() { return } + // Print remove frequency if provided + if *minimum > 0 { + fmt.Fprintf(os.Stderr, "[*] Removing items with frequency less than %d.\n", *minimum) + } + // Remove items under minimum frequency if provided if *minimum > 0 { primaryMap = format.RemoveMinimumFrequency(primaryMap, *minimum) } + // Print length range if provided + if lenRange.Start > 0 || lenRange.End > 0 { + fmt.Fprintf(os.Stderr, "[*] Only outputting items between %d and %d characters.\n", lenRange.Start, lenRange.End) + } + // Remove items outside of length range if provided if lenRange.Start > 0 || lenRange.End > 0 { primaryMap = format.RemoveLengthRange(primaryMap, lenRange.Start, lenRange.End) } + // Print retained and removed items if provided + if len(retainMap) > 0 || len(removeMap) > 0 { + fmt.Fprintf(os.Stderr, "[*] Retain/remove flags provided. Retaining %d and removing %d items.\n", len(retainMap), len(removeMap)) + } + // Process retain and remove maps if provided if len(retainMap) > 0 || len(removeMap) > 0 { primaryMap, err = format.RetainRemove(primaryMap, retainMap, removeMap, *debugMode) @@ -188,15 +208,25 @@ func main() { } } + // if -n is providied, filter ALL results to only that top amount + if *outputVerboseMax > 0 { + primaryMap = format.FilterTopN(primaryMap, *outputVerboseMax) + } + // Print output to stdout if *verbose3 { - format.PrintStatsToSTDOUT(primaryMap, *verbose3, *verboseStatsMax) + format.PrintStatsToSTDOUT(primaryMap, *verbose3, *outputVerboseMax) } else if *verbose2 { - format.PrintStatsToSTDOUT(primaryMap, *verbose3, *verboseStatsMax) + format.PrintStatsToSTDOUT(primaryMap, *verbose3, *outputVerboseMax) } else { format.PrintArrayToSTDOUT(primaryMap, *verbose) } + // Print output location if provided + if *jsonOutput != "" { + fmt.Fprintf(os.Stderr, "[*] Saving output to JSON file: %s\n", *jsonOutput) + } + // Save output to JSON if provided if *jsonOutput != "" { err = format.SaveArrayToJSON(*jsonOutput, primaryMap) diff --git a/pkg/format/format.go b/pkg/format/format.go index b4de445..e313c61 100644 --- a/pkg/format/format.go +++ b/pkg/format/format.go @@ -77,6 +77,10 @@ func PrintStatsToSTDOUT(freq map[string]int, verbose bool, max int) { sort.Sort(sort.Reverse(p)) sort.Sort(sort.Reverse(normalizedP)) + if max == 0 { + max = 25 + } + if max > len(p) { max = len(p) } @@ -528,6 +532,33 @@ func RemoveLengthRange(freq map[string]int, start int, end int) map[string]int { return newFreq } +// FilterTopN removes all but the top N items from a map of item frequencies +// and returns a new map +// +// Args: +// freq (map[string]int): A map of item frequencies +// n (int): The number of items to retain +// +// Returns: +// map[string]int: A new map of the top N item frequencies +func FilterTopN(freq map[string]int, n int) map[string]int { + newFreq := make(map[string]int) + p := make(models.PairList, len(freq)) + i := 0 + for k, v := range freq { + p[i] = models.Pair{k, v} + i++ + } + sort.Sort(sort.Reverse(p)) + if n > len(p) { + n = len(p) + } + for i := 0; i < n; i++ { + newFreq[p[i].Key] = p[i].Value + } + return newFreq +} + // ---------------------------------------------------------------------------- // Encoding Functions // ---------------------------------------------------------------------------- diff --git a/pkg/format/format_test.go b/pkg/format/format_test.go index 7d08a90..401bd64 100644 --- a/pkg/format/format_test.go +++ b/pkg/format/format_test.go @@ -13,6 +13,7 @@ import ( // - RetainRemove() // - RemoveMinimumFrequency() // - RemoveLengthRange() +// - FilterTopN() // // ** Encoding Functions ** // - EncodeInputMap() @@ -153,6 +154,35 @@ func TestRemoveLengthRange(t *testing.T) { } } +// Unit Test for FilterTopN() +func TestFilterTopN(t *testing.T) { + + // Define a test case struct + type testCase struct { + input map[string]int + top int + output map[string]int + } + + type testCases []testCase + + // Define a test case + tests := testCases{ + {map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}, 2, map[string]int{"c": 3, "d": 4}}, + {map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}, 3, map[string]int{"b": 2, "c": 3, "d": 4}}, + {map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}, 4, map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}}, + {map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}, 5, map[string]int{"a": 1, "b": 2, "c": 3, "d": 4}}, + } + + // Run test cases + for _, test := range tests { + result := FilterTopN(test.input, test.top) + if utils.CheckAreMapsEqual(result, test.output) == false { + t.Errorf("FilterTopN() failed - expected: %v, got: %v", test.output, result) + } + } +} + // Unit Test for EncodeInputMap() func TestEncodeInputMap(t *testing.T) { diff --git a/pkg/mask/mask.go b/pkg/mask/mask.go index f5c46a6..17d1ec7 100644 --- a/pkg/mask/mask.go +++ b/pkg/mask/mask.go @@ -352,7 +352,9 @@ func MakeMatchedMaskedMap(input map[string]int, replacementMask string, maskMap } } case true: - fmt.Println(newKey) + if _, exists := maskMap[newKey]; exists { + fmt.Println(key) + } } } return maskedMap diff --git a/pkg/transform/transform.go b/pkg/transform/transform.go index 1d1236e..312457e 100644 --- a/pkg/transform/transform.go +++ b/pkg/transform/transform.go @@ -99,7 +99,7 @@ func TransformationController(input map[string]int, mode string, startingIndex i os.Exit(1) } output = mask.MakeMatchedMaskedMap(input, replacementMask, transformationFilesMap, bypass, functionDebug) - case "swap": + case "swap", "swap-single": if len(transformationFilesMap) == 0 { fmt.Fprintf(os.Stderr, "[!] Swap operations require use of one or more -tf flags to specify one or more files\n") fmt.Fprintf(os.Stderr, "[!] This transformation mode requires a ':' separated list of keys to swap\n") @@ -123,7 +123,7 @@ func TransformationController(input map[string]int, mode string, startingIndex i output = MakePassphraseMap(input, transformationFilesMap, bypass, functionDebug, passphraseWords) case "substring": output = utils.SubstringMap(input, startingIndex, endingIndex, bypass, functionDebug) - case "replace": + case "replace-all", "replace": if len(transformationFilesMap) == 0 { fmt.Fprintf(os.Stderr, "[!] Replace operations require use of one or more -tf flags to specify one or more files\n") os.Exit(1)