Skip to content

Commit

Permalink
Adds support for removing banned words and compromised passwords from…
Browse files Browse the repository at this point in the history 
… the store
  • Loading branch information
@ryannewington
ryannewington committed Jun 17, 2019
1 parent 8604e38 commit f37741b
Show file tree
Hide file tree
Showing 14 changed files with 264 additions and 118 deletions.
2 changes: 2 additions & 0 deletions src/Lithnet.ActiveDirectory.PasswordProtection.sln
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ Global
{850894DF-B439-4DAA-A574-F7CFEA422A2F}.Release|x86.Build.0 = Release|Win32
{F42EF09F-7AB1-484B-BF80-72D455E9E9E4}.Debug|Any CPU.ActiveCfg = Debug|x86
{F42EF09F-7AB1-484B-BF80-72D455E9E9E4}.Debug|x64.ActiveCfg = Debug|x64
{F42EF09F-7AB1-484B-BF80-72D455E9E9E4}.Debug|x64.Build.0 = Debug|x64
{F42EF09F-7AB1-484B-BF80-72D455E9E9E4}.Debug|x86.ActiveCfg = Debug|x86
{F42EF09F-7AB1-484B-BF80-72D455E9E9E4}.Debug|x86.Build.0 = Debug|x86
{F42EF09F-7AB1-484B-BF80-72D455E9E9E4}.Release|Any CPU.ActiveCfg = Release|x86
Expand All @@ -55,6 +56,7 @@ Global
{B94AE2C5-72F3-40FC-BB70-C18B33D19A2B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{B94AE2C5-72F3-40FC-BB70-C18B33D19A2B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{B94AE2C5-72F3-40FC-BB70-C18B33D19A2B}.Debug|x64.ActiveCfg = Debug|x64
{B94AE2C5-72F3-40FC-BB70-C18B33D19A2B}.Debug|x64.Build.0 = Debug|x64
{B94AE2C5-72F3-40FC-BB70-C18B33D19A2B}.Debug|x86.ActiveCfg = Debug|Any CPU
{B94AE2C5-72F3-40FC-BB70-C18B33D19A2B}.Debug|x86.Build.0 = Debug|Any CPU
{B94AE2C5-72F3-40FC-BB70-C18B33D19A2B}.Release|Any CPU.ActiveCfg = Release|Any CPU
Expand Down
16 changes: 0 additions & 16 deletions src/ManagedUnitTests/LsassTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,21 +81,5 @@ public void TestDurationPolicyDisabled()
key.SetValue("Disabled", 0);
}
}

[TestMethod]
public void TestEmptyGuid()
{
using (PrincipalContext ctx = new PrincipalContext(ContextType.Machine))
{
UserPrincipal p = UserPrincipal.FindByIdentity(ctx, "pwntest");

if (p == null)
{
Assert.Fail("The test user was not found");
}

p.SetPassword(new Guid().ToString());
}
}
}
}
4 changes: 2 additions & 2 deletions src/ManagedUnitTests/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,5 +16,5 @@
[assembly: Guid("b94ae2c5-72f3-40fc-bb70-c18b33d19a2b")]

// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.0.7107.3289")]
[assembly: AssemblyFileVersion("1.0.7107.3289")]
[assembly: AssemblyVersion("1.0.7107.8661")]
[assembly: AssemblyFileVersion("1.0.7107.8661")]
150 changes: 60 additions & 90 deletions src/ManagedUnitTests/V3StoreTests.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,14 +32,6 @@ protected override string GetPrefixFromHash(string hash)
return hash.Substring(0, 4);
}

[TestMethod]
public void TestHashFileIsInOrder()
{
Assert.IsTrue(Lithnet.ActiveDirectory.PasswordProtection.Store.DoesHexHashFileAppearSorted(@"D:\pwnedpwds\raw\pwned-passwords-ntlm-ordered-by-hash.txt", 16));
Assert.IsFalse(Lithnet.ActiveDirectory.PasswordProtection.Store.DoesHexHashFileAppearSorted(@"D:\pwnedpwds\raw\pwned-passwords-ntlm-ordered-by-count.txt", 16));
}


[TestMethod]
public void TestGoodHashTypes()
{
Expand Down Expand Up @@ -72,97 +64,24 @@ public void TestHashTooShort()
}
}


[TestMethod]
public void BuildUsablev3Store()
public void TestAddNormalizedWordToStore()
{
return;
string path = Path.Combine(TestHelpers.TestStorePath, "v3Build");
Directory.CreateDirectory(path);
V3Store store = new V3Store(path);

CancellationTokenSource ct = new CancellationTokenSource();

// Start with HIBP
string file = @"D:\pwnedpwds\raw\pwned-passwords-ntlm-ordered-by-hash.txt";
Lithnet.ActiveDirectory.PasswordProtection.Store.ImportHexHashesFromSortedFile(store, StoreType.Password, file, ct.Token);

// add english dictionary to word store
file = @"D:\pwnedpwds\raw\english.txt";
Lithnet.ActiveDirectory.PasswordProtection.Store.ImportPasswordsFromFile(store, StoreType.Word, file, ct.Token);

// add more english words to word store
file = @"D:\pwnedpwds\raw\words.txt";
Lithnet.ActiveDirectory.PasswordProtection.Store.ImportPasswordsFromFile(store, StoreType.Word, file, ct.Token);
string password = "password"; //8846F7EAEE8FB117AD06BDD830B7586C

// add rockyou breach
file = @"D:\pwnedpwds\raw\rockyou.txt";
Lithnet.ActiveDirectory.PasswordProtection.Store.ImportPasswordsFromFile(store, StoreType.Password, file, ct.Token);
this.Store.AddToStore(password, StoreType.Word);

// add top 100000
file = @"D:\pwnedpwds\raw\top1000000.txt";
Lithnet.ActiveDirectory.PasswordProtection.Store.ImportPasswordsFromFile(store, StoreType.Password, file, ct.Token);
string rawFile = Path.Combine(this.Store.StorePathWordStore, this.GetFileNameFromHash("8846F7EAEE8FB117AD06BDD830B7586C"));
TestHelpers.AssertFileIsExpectedSize(rawFile, this.StoredHashSize);

// add breach compilation
file = @"D:\pwnedpwds\raw\breachcompilationuniq.txt";
Lithnet.ActiveDirectory.PasswordProtection.Store.ImportPasswordsFromFile(store, StoreType.Password, file, ct.Token);
}
this.Store.AddToStore(password, StoreType.Word);

[TestMethod]
public void TestBadPassword()
{
V3Store store = new V3Store(@"D:\pwnedpwds\store");
Assert.IsTrue(store.IsInStore("password!!!!", StoreType.Word));
}
TestHelpers.AssertFileIsExpectedSize(rawFile, this.StoredHashSize);

[TestMethod]
public void TestBadPassword2()
{
V3Store store = new V3Store(@"D:\pwnedpwds\store");
Assert.IsTrue(store.IsInStore("Password345!", StoreType.Word));
Assert.IsTrue(this.Store.IsInStore(password, StoreType.Word));
Assert.IsTrue(this.Store.IsInStore("Password1234!", StoreType.Word));
}

//[TestMethod]
//public void BuildStoreEnglish()
//{
// this.BuildStore(@"D:\pwnedpwds\raw\english.txt");
//}

//[TestMethod]
//public void BuildStoreWords()
//{
// this.BuildStore(@"D:\pwnedpwds\raw\words.txt");
//}

//[TestMethod]
//public void BuildStoreRockyou()
//{
// this.BuildStore(@"D:\pwnedpwds\raw\rockyou.txt");
//}

//[TestMethod]
//public void BuildStoreTop1000000()
//{
// this.BuildStore(@"D:\pwnedpwds\raw\top1000000.txt");
//}

//[TestMethod]
//public void BuildStoreBreachCompilationUniq()
//{
// this.BuildStore(@"D:\pwnedpwds\raw\breachcompilationuniq.txt");
//}

//[TestMethod]
//public void BuildStoreHibp()
//{
// string file = @"D:\pwnedpwds\raw\pwned-passwords-ntlm-ordered-by-hash.txt";
// string path = Path.Combine(TestHelpers.TestStorePath, "hibp");
// Directory.CreateDirectory(path);

// var store = new V3Store(path);
// StoreInterface.Store.ImportHexHashesFromSortedFile(store, file);
//}

[TestMethod]
public void AddEnglishDictionaryToNewStoreAndValidate()
{
Expand Down Expand Up @@ -220,6 +139,57 @@ public void TestAddPasswordToStore()
Assert.IsTrue(this.Store.IsInStore(password, StoreType.Password));
}

[TestMethod]
public void TestRemovePasswordFromStore()
{
string password = "password"; //8846F7EAEE8FB117AD06BDD830B7586C

this.Store.AddToStore(password, StoreType.Password);

string rawFile = Path.Combine(this.Store.StorePathPasswordStore, this.GetFileNameFromHash("8846F7EAEE8FB117AD06BDD830B7586C"));
TestHelpers.AssertFileIsExpectedSize(rawFile, this.StoredHashSize);

Assert.IsTrue(this.Store.IsInStore(password, StoreType.Password));

this.Store.RemoveFromStore(password, StoreType.Password);
TestHelpers.AssertFileIsExpectedSize(rawFile, 0);
Assert.IsFalse(this.Store.IsInStore(password, StoreType.Password));
}

[TestMethod]
public void TestRemoveWordFromStore()
{
string password = "password"; //8846F7EAEE8FB117AD06BDD830B7586C

this.Store.AddToStore(password, StoreType.Word);

string rawFile = Path.Combine(this.Store.StorePathWordStore, this.GetFileNameFromHash("8846F7EAEE8FB117AD06BDD830B7586C"));
TestHelpers.AssertFileIsExpectedSize(rawFile, this.StoredHashSize);

Assert.IsTrue(this.Store.IsInStore(password, StoreType.Word));

this.Store.RemoveFromStore(password, StoreType.Word);
TestHelpers.AssertFileIsExpectedSize(rawFile, 0);
Assert.IsFalse(this.Store.IsInStore(password, StoreType.Word));
}

[TestMethod]
public void TestRemoveNormalizedWordFromStore()
{
string password = "password"; //8846F7EAEE8FB117AD06BDD830B7586C

this.Store.AddToStore(password, StoreType.Word);

string rawFile = Path.Combine(this.Store.StorePathWordStore, this.GetFileNameFromHash("8846F7EAEE8FB117AD06BDD830B7586C"));
TestHelpers.AssertFileIsExpectedSize(rawFile, this.StoredHashSize);

Assert.IsTrue(this.Store.IsInStore(password, StoreType.Word));

this.Store.RemoveFromStore("password1", StoreType.Word);
TestHelpers.AssertFileIsExpectedSize(rawFile, 0);
Assert.IsFalse(this.Store.IsInStore(password, StoreType.Word));
}

[TestMethod]
public void TestAddHashToStore()
{
Expand Down
Binary file modified src/PasswordFilter/PasswordFilter.rc
View file
Binary file not shown.
5 changes: 5 additions & 0 deletions src/PasswordProtection/BinaryStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,11 @@ protected override void AddToStore(HashSet<byte[]> hashes, string range, StoreTy
this.GetInstance(storeType).AddHashRangeToStore(hashes, range, progress);
}

protected override void RemoveFromStore(HashSet<byte[]> hashes, string range, StoreType storeType, OperationProgress progress)
{
this.GetInstance(storeType).RemoveHashRangeFromStore(hashes, range, progress);
}

public override bool IsInStore(byte[] hash, StoreType storeType)
{
return this.GetInstance(storeType).IsHashInStore(hash);
Expand Down
34 changes: 33 additions & 1 deletion src/PasswordProtection/BinaryStoreInstance.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

namespace Lithnet.ActiveDirectory.PasswordProtection
{
public class BinaryStoreInstance
internal class BinaryStoreInstance
{
public bool IsInBatch { get; private set; }

Expand Down Expand Up @@ -118,6 +118,38 @@ public void AddHashRangeToStore(HashSet<byte[]> incomingHashes, string range, Op
}
}

public void RemoveHashRangeFromStore(HashSet<byte[]> hashesToRemove, string range, OperationProgress progress)
{
if (hashesToRemove == null)
{
throw new ArgumentNullException(nameof(hashesToRemove));
}

if (range == null)
{
throw new ArgumentNullException(nameof(range));
}

string file = Path.Combine(this.StorePath, $"{range}.db");

bool hasChanges = false;
HashSet<byte[]> storedHashes = new HashSet<byte[]>(ByteArrayComparer.Comparer);

if (File.Exists(file))
{
this.LoadHashesFromStoreFile(file, storedHashes, progress);
foreach (byte[] hashToRemove in hashesToRemove)
{
hasChanges |= storedHashes.Remove(hashToRemove);
}
}

if (hasChanges)
{
this.WriteStoreFile(file, false, storedHashes);
}
}

public string GetRangeFromHash(string hash)
{
if (hash == null)
Expand Down
12 changes: 9 additions & 3 deletions src/PasswordProtection/IStore.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,18 @@ public interface IStore
{
void ClearStore(StoreType storeType);

void RemoveFromStore(string password, StoreType storeType);

void RemoveFromStore(byte[] hash, StoreType storeType);

void RemoveFromStore(HashSet<byte[]> hashes, StoreType storeType, CancellationToken ct, OperationProgress progress);

void AddToStore(string password, StoreType storeType);

void AddToStore(byte[] hash, StoreType storeType);

void AddToStore(HashSet<byte[]> hashes, StoreType storeType, CancellationToken ct, OperationProgress progress);

bool IsInStore(string password, StoreType storeType);

bool IsInStore(byte[] hash, StoreType storeType);
Expand Down
6 changes: 3 additions & 3 deletions src/PasswordProtection/Properties/AssemblyInfo.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,6 @@
//
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.7107.3289")]
[assembly: AssemblyVersion("1.0.7107.3289")]
[assembly: AssemblyFileVersion("1.0.7107.3289")]
// [assembly: AssemblyVersion("1.0.7107.8661")]
[assembly: AssemblyVersion("1.0.7107.8661")]
[assembly: AssemblyFileVersion("1.0.7107.8661")]
61 changes: 61 additions & 0 deletions src/PasswordProtection/Store.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,6 +337,35 @@ private static IEnumerable<string> GetLinesFromFile(string sourceFile, Operation
progress.FileReadInProgress = false;
}

public void RemoveFromStore(byte[] hash, StoreType storeType)
{
HashSet<byte[]> hashset = new HashSet<byte[]>(ByteArrayComparer.Comparer);
hashset.Add(hash);
this.RemoveFromStore(hashset, storeType, new CancellationToken(), new OperationProgress());
}

public void RemoveFromStore(string password, StoreType storeType)
{
if (storeType == StoreType.Word)
{
password = StringNormalizer.Normalize(password);
}

this.RemoveFromStore(this.ComputeHash(password), storeType);
}

public void RemoveFromStore(HashSet<byte[]> hashes, StoreType storeType, CancellationToken ct, OperationProgress progress)
{
this.RemoveFromStore(
hashes
.GroupBy(this.GetRangeFromHash, StringComparer.OrdinalIgnoreCase)
.ToDictionary(g => g.Key, g => new HashSet<byte[]>(g, ByteArrayComparer.Comparer)),
storeType,
ct,
false,
progress);
}

public void AddToStore(string password, StoreType storeType)
{
if (storeType == StoreType.Word)
Expand Down Expand Up @@ -395,6 +424,36 @@ private void AddToStore(Dictionary<string, HashSet<byte[]>> hashes, StoreType st
}
}

private void RemoveFromStore(Dictionary<string, HashSet<byte[]>> hashes, StoreType storeType, CancellationToken ct, bool emptyAfterCommit, OperationProgress progress)
{
try
{
ParallelOptions o = new ParallelOptions();
o.CancellationToken = ct;
o.MaxDegreeOfParallelism = Debugger.IsAttached ? 1 : -1;

progress.FlushStoreInProgress = true;
progress.FlushStoreTotal = hashes.Count;
progress.FlushStorePosition = 0;
progress.FlushStoreStartTime = DateTime.Now;

Parallel.ForEach(hashes, o, group =>
{
this.RemoveFromStore(group.Value, group.Key, storeType, progress);
progress.IncrementFlushStorePosition();
if (emptyAfterCommit)
{
group.Value.Clear();
}
});
}
finally
{
progress.FlushStoreInProgress = false;
}
}


public bool IsInStore(string password, StoreType storeType)
{
if (storeType == StoreType.Word)
Expand All @@ -419,6 +478,8 @@ public bool IsInStore(string password, StoreType storeType)

protected abstract void AddToStore(HashSet<byte[]> hashes, string range, StoreType storeType, OperationProgress progress);

protected abstract void RemoveFromStore(HashSet<byte[]> hashes, string range, StoreType storeType, OperationProgress progress);

public abstract void StartBatch(StoreType storeType);

public abstract byte[] ComputeHash(string text);
Expand Down
Loading

0 comments on commit f37741b

Please sign in to comment.