diff --git a/BitSwarm (Console Core Demo)/BitSwarm (Console Core Demo).csproj b/BitSwarm (Console Core Demo)/BitSwarm (Console Core Demo).csproj index 77159d4..d1800a9 100644 --- a/BitSwarm (Console Core Demo)/BitSwarm (Console Core Demo).csproj +++ b/BitSwarm (Console Core Demo)/BitSwarm (Console Core Demo).csproj @@ -5,9 +5,9 @@ netcoreapp3.1 BitSwarmConsole bswarm - 2.2.6.0 - 2.2.6.0 - 2.2.6 + 2.2.7.0 + 2.2.7.0 + 2.2.7 diff --git a/BitSwarm (Console Core Demo)/Program.cs b/BitSwarm (Console Core Demo)/Program.cs index d779f3a..926ff7e 100644 --- a/BitSwarm (Console Core Demo)/Program.cs +++ b/BitSwarm (Console Core Demo)/Program.cs @@ -15,9 +15,7 @@ class Program static bool sessionFinished = false; static bool preventOnce = true; - static bool resized = false; - static int consoleLastTop = -1; - static int prevHeight; + static int consoleStatsPos = 0; static void Main(string[] args) { @@ -42,12 +40,7 @@ static void Main(string[] args) // Prepare Options opt = new BitSwarm.DefaultOptions(); - if (args.Length >= 2) - { - if (!Directory.Exists(args[1])) Directory.CreateDirectory(args[1]); - opt.DownloadPath = args[1]; - } - + if (args.Length >= 2) opt.DownloadPath = args[1]; if (args.Length >= 3) opt.MinThreads = int.Parse(args[2]); if (args.Length >= 4) opt.MaxThreads = int.Parse(args[3]); if (args.Length >= 5) opt.SleepModeLimit = int.Parse(args[4]); @@ -63,21 +56,25 @@ static void Main(string[] args) // More Options + // [Feeders] + //opt.EnablePEX = false; //opt.EnableDHT = false; //opt.EnableTrackers= false; //opt.TrackersPath = @"c:\root\trackers.txt"; + // [Timeouts] //opt.ConnectionTimeout = 1200; //opt.HandshakeTimeout = 2400; //opt.MetadataTimeout = 1600; - //opt.PieceTimeout = 7000; + //opt.PieceTimeout = 2000; + //opt.PieceRetries = 3; // Re-requests timed-out pieces on the first timeout // Initialize BitSwarm bitSwarm = new BitSwarm(opt); - bitSwarm.StatsUpdated += BitSwarm_StatsUpdated; - bitSwarm.MetadataReceived += BitSwarm_MetadataReceived; - bitSwarm.StatusChanged += BitSwarm_StatusChanged; + bitSwarm.MetadataReceived += BitSwarm_MetadataReceived; // Receives torrent data [on torrent file will fire directly, on magnetlink will fire on metadata received] + bitSwarm.StatsUpdated += BitSwarm_StatsUpdated; // Stats refresh every 2 seconds + bitSwarm.StatusChanged += BitSwarm_StatusChanged; // Paused/Stopped or Finished if (File.Exists(args[0])) bitSwarm.Initiliaze(args[0]); @@ -88,13 +85,8 @@ static void Main(string[] args) bitSwarm.Start(); Console.CancelKeyPress += new ConsoleCancelEventHandler(CtrlC); - prevHeight = Console.WindowHeight; - while (!sessionFinished) - { - if (Console.WindowHeight != prevHeight) { prevHeight = Console.WindowHeight; resized = true; } - Thread.Sleep(500); - } + while (!sessionFinished) Thread.Sleep(500); } protected static void CtrlC(object sender, ConsoleCancelEventArgs args) { @@ -121,11 +113,20 @@ private static void BitSwarm_MetadataReceived(object source, BitSwarm.MetadataRe { torrent = e.Torrent; Console.Clear(); + if (Utils.IsWindows) { Console.WriteLine(bitSwarm.DumpTorrent() + "\r\n"); consoleStatsPos = Console.CursorTop; } } private static void BitSwarm_StatsUpdated(object source, BitSwarm.StatsUpdatedArgs e) { - Console.SetCursorPosition(0, 0); - Console.WriteLine(bitSwarm.DumpTorrent() + "\r\n"); + if (Utils.IsWindows) + { + Console.SetCursorPosition(0, consoleStatsPos); + } + else + { + Console.SetCursorPosition(0, 0); + Console.WriteLine(bitSwarm.DumpTorrent() + "\r\n"); + } + Console.WriteLine(bitSwarm.DumpStats()); } } diff --git a/BitSwarm (WinForms Demo)/Properties/AssemblyInfo.cs b/BitSwarm (WinForms Demo)/Properties/AssemblyInfo.cs index 5b319c4..ada4968 100644 --- a/BitSwarm (WinForms Demo)/Properties/AssemblyInfo.cs +++ b/BitSwarm (WinForms Demo)/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.2.6.0")] -[assembly: AssemblyFileVersion("2.2.6.0")] +[assembly: AssemblyVersion("2.2.7.0")] +[assembly: AssemblyFileVersion("2.2.7.0")] diff --git a/BitSwarm/BitSwarm.cs b/BitSwarm/BitSwarm.cs index dcf2316..764bd45 100644 --- a/BitSwarm/BitSwarm.cs +++ b/BitSwarm/BitSwarm.cs @@ -2,45 +2,15 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; -using System.Linq; using System.Threading; using System.Security.Cryptography; +using BencodeNET.Objects; using SuRGeoNix.BEP; using static SuRGeoNix.BEP.Torrent.TorrentData; namespace SuRGeoNix { - /* TODO (Draft Notes) - * - * 1. [Jobs -> Peer] | TimeBeginPeriod(1) is too much, after that we could go in TimeBeginPeriod(>=5) or even remove it completely - * Try to avoid all the schedule jobs and transfer the functionality directly in the Peer.ProcessMessage() - * eg. Piece Request after Unchoked, PieceReceived, PieceRejected - * Similarly should handle possible Piece Cancel/Timeouts/KeepAlives/Interested/"Autokill/drop" etc. - * - * Additionally, check where possible to Dispatch New Peers when they arrive (fill from trackers/fill from dht etc.) - * Randomness on how you dispatch them? (to avoid upload limits and high cpu?) - * - * 2. [Peers Lists] - * Αvoid large lists (ban peers, less re-fills) - * - * 3. [Streaming] | Focus Points Requests Algorithm - * The problem: We need as soon as possible specific pieces so by requesting the same pieces/blocks from more than one peer, - * we ensure we will receive it faster but at the same time we will have more already received pieces/blocks => dropped bytes - * - * The right adjustment it is required for Piece Timeout, RequestBlocksPerPeer, Piece/Block Randomness - * - * 4. [Threads] - * Transfer all the threads (especially .net threadpool) to BSTP - * Review also async & thread sleeps (try to replace them with better alternatives) - * - * 5. [Sleep Mode] - * - * Consider using an automatic approach (eg. after X seconds of current session get [Max Down Rate - 700KB/s] - * - * 6. +++ Allow Fast / PeX / uTP / Proxy / Seeding / Load-Save Session / Multiple-Instances - */ - public class BitSwarm { #region BitSwarm's Thread Pool [Short/Long Run] @@ -113,7 +83,8 @@ private void ThreadRun(int index) { Threads[index].peer?.Run(this, index); if (ShortRun > MinThreads || Stop || Threads == null || Threads[index] == null) break; - if (peersForDispatch.TryPop(out Peer tmp)) { Threads[index].peer = tmp; Threads[index].peer.status = Peer.Status.CONNECTING; } else break; + lock (peersForDispatch) + if (peersForDispatch.TryPop(out Peer tmp)) { Threads[index].peer = tmp; Threads[index].peer.status = Peer.Status.CONNECTING; } else break; } while (true); @@ -227,6 +198,7 @@ public class DefaultOptions public int PieceTimeout { get; set; } = 5000; public int PieceRetries { get; set; } = 0; + public bool EnablePEX { get; set; } = true; public bool EnableDHT { get; set; } = true; public bool EnableTrackers { get; set; } = true; public int PeersFromTracker { get; set; } = -1; @@ -304,9 +276,8 @@ private enum Status internal enum PeersStorage { DHT, - TRACKERS, - DHTNEW, - TRACKERSNEW + PEX, + TRACKER } #endregion @@ -405,6 +376,7 @@ public StatsUpdatedArgs(StatsStructure stats) private int sha1Fails = 0; public int dhtPeers = 0; // Not accurate (based on uniqueness) public int trackersPeers = 0; // Not accurate (based on uniqueness) + public int pexPeers = 0; // Not accurate (based on uniqueness) #endregion @@ -424,6 +396,8 @@ public void Initiliaze(Uri magnetLink) } private void Initiliaze() { + if (!Directory.Exists(Options.DownloadPath)) Directory.CreateDirectory(Options.DownloadPath); + bstp = new BSTP(); status = Status.STOPPED; @@ -468,8 +442,13 @@ private void Setup() trackerOpt.Verbosity = Options.LogTracker ? Options.Verbosity : 0; // Peer - Peer.Beggar = this; + BDictionary extensions = new BDictionary(); + extensions.Add("ut_metadata", Peer.Messages.EXT_UT_METADATA); + if (Options.EnablePEX) extensions.Add("ut_pex", Peer.Messages.EXT_UT_PEX); + Peer.HANDSHAKE_BYTES = Utils.ArrayMerge(Peer.BIT_PROTO, Peer.EXT_PROTO, Utils.StringHexToArray(torrent.file.infoHash), peerID); + Peer.EXT_BDIC = (new BDictionary() { {"e", 0 }, { "m", extensions }, {"reqq", 250 } }).EncodeAsBytes(); + Peer.Beggar = this; // Fill from TorrentFile | MagnetLink + TrackersPath to Trackers torrent.FillTrackersFromTrackersPath(Options.TrackersPath); @@ -640,16 +619,16 @@ internal void FillPeers(Dictionary newPeers, PeersStorage storage) int countNew = 0; lock (peersForDispatch) - foreach (KeyValuePair peerKV in newPeers) - { - if (!peersStored.ContainsKey(peerKV.Key) && !peersBanned.Contains(peerKV.Key)) + foreach (KeyValuePair peerKV in newPeers) { - peersStored.TryAdd(peerKV.Key, peerKV.Value); - Peer peer = new Peer(peerKV.Key, peerKV.Value); - peersForDispatch.Push(peer); //if (!bstp.Dispatch(peer)) peersForDispatch.Push(peer); - countNew++; + if (!peersStored.ContainsKey(peerKV.Key) && !peersBanned.Contains(peerKV.Key)) + { + peersStored.TryAdd(peerKV.Key, peerKV.Value); + Peer peer = new Peer(peerKV.Key, peerKV.Value); + peersForDispatch.Push(peer); //if (!bstp.Dispatch(peer)) peersForDispatch.Push(peer); + countNew++; + } } - } if (peersForDispatch.Count > 0 && bstp.ShortRun < bstp.MinThreads) { @@ -657,10 +636,12 @@ internal void FillPeers(Dictionary newPeers, PeersStorage storage) bstp.Dispatch(null); } - if (storage == PeersStorage.TRACKERSNEW) + if (storage == PeersStorage.TRACKER) trackersPeers += countNew; - else if (storage == PeersStorage.DHTNEW) + else if (storage == PeersStorage.DHT) dhtPeers += countNew; + else if (storage == PeersStorage.PEX) + pexPeers += countNew; if (Options.Verbosity > 0 && countNew > 0) Log($"[{storage.ToString()}] {countNew} Adding Peers"); } @@ -699,94 +680,91 @@ internal void ReFillPeers() private void FillStats() { - // Stats - Stats.CurrentTime = DateTime.UtcNow.Ticks; - - // Progress - int includedSetsCounter = torrent.data.progress.setsCounter - (torrent.data.pieces - Stats.PiecesIncluded); - Stats.BytesCurDownloaded = includedSetsCounter * torrent.data.pieceSize; - if (Stats.BytesCurDownloaded < 0) Stats.BytesCurDownloaded = 0; - Stats.Progress = includedSetsCounter > 0 ? (int) (includedSetsCounter * 100.0 / Stats.PiecesIncluded) : 0; - - // Rates - double secondsDiff = ((Stats.CurrentTime - prevStatsTicks) / 10000000.0); // For more accurancy - long totalBytesDownloaded = Stats.BytesDownloaded + Stats.BytesDropped; // Included or Not? - Stats.DownRate = (int) ((totalBytesDownloaded - Stats.BytesDownloadedPrev) / secondsDiff); // Change this (2 seconds) if you change scheduler - Stats.AvgRate = (int) ( totalBytesDownloaded / curSecond); - if (Stats.DownRate > Stats.MaxRate) Stats.MaxRate = Stats.DownRate; - - // ETAs - if (torrent.data.pieces != Stats.PiecesIncluded && curSecond > 0 && Stats.BytesCurDownloaded > 1) - { - Stats.AvgETA = (int) ( (Stats.PiecesIncluded * torrent.data.pieceSize) / (Stats.BytesCurDownloaded / curSecond ) ); - } - else + try { - if (torrent.data.totalSize - Stats.BytesDownloaded == 0) + // Stats + Stats.CurrentTime = DateTime.UtcNow.Ticks; + + // Progress + int includedSetsCounter = torrent.data.progress.setsCounter - (torrent.data.pieces - Stats.PiecesIncluded); + Stats.BytesCurDownloaded = includedSetsCounter * torrent.data.pieceSize; + if (Stats.BytesCurDownloaded < 0) Stats.BytesCurDownloaded = 0; + Stats.Progress = includedSetsCounter > 0 ? (int) (includedSetsCounter * 100.0 / Stats.PiecesIncluded) : 0; + + // Rates + double secondsDiff = ((Stats.CurrentTime - prevStatsTicks) / 10000000.0); // For more accurancy + long totalBytesDownloaded = Stats.BytesDownloaded + Stats.BytesDropped; // Included or Not? + Stats.DownRate = (int) ((totalBytesDownloaded - Stats.BytesDownloadedPrev) / secondsDiff); // Change this (2 seconds) if you change scheduler + Stats.AvgRate = (int) ( totalBytesDownloaded / curSecond); + if (Stats.DownRate > Stats.MaxRate) Stats.MaxRate = Stats.DownRate; + + // ETAs + if (torrent.data.pieces != Stats.PiecesIncluded && curSecond > 0 && Stats.BytesCurDownloaded > 1) { - Stats.ETA = 0; - Stats.AvgETA = 0; - } + Stats.AvgETA = (int) ( (Stats.PiecesIncluded * torrent.data.pieceSize) / (Stats.BytesCurDownloaded / curSecond ) ); + } else { - if (Stats.BytesDownloaded - Stats.BytesDownloadedPrev == 0) - Stats.ETA *= 2; // Kind of infinite | int overflow nice - else - Stats.ETA = (int) ( (torrent.data.totalSize - Stats.BytesDownloaded) / ((Stats.BytesDownloaded - Stats.BytesDownloadedPrev) / secondsDiff) ); + if (torrent.data.totalSize - Stats.BytesDownloaded == 0) + { + Stats.ETA = 0; + Stats.AvgETA = 0; + } + else + { + if (Stats.BytesDownloaded - Stats.BytesDownloadedPrev == 0) + Stats.ETA *= 2; // Kind of infinite | int overflow nice + else + Stats.ETA = (int) ( (torrent.data.totalSize - Stats.BytesDownloaded) / ((Stats.BytesDownloaded - Stats.BytesDownloadedPrev) / secondsDiff) ); - if (Stats.BytesDownloaded != 0 && curSecond > 0 && Stats.BytesDownloaded > 1) - Stats.AvgETA = (int) ( (torrent.data.totalSize - Stats.BytesDownloaded) / (Stats.BytesDownloaded / curSecond ) ); + if (Stats.BytesDownloaded != 0 && curSecond > 0 && Stats.BytesDownloaded > 1) + Stats.AvgETA = (int) ( (torrent.data.totalSize - Stats.BytesDownloaded) / (Stats.BytesDownloaded / curSecond ) ); + } } - } - // Save Cur to Prev - Stats.BytesDownloadedPrev = totalBytesDownloaded; - prevStatsTicks = Stats.CurrentTime; + // Save Cur to Prev + Stats.BytesDownloadedPrev = totalBytesDownloaded; + prevStatsTicks = Stats.CurrentTime; - // Stats & Clean-up - Stats.PeersTotal = peersStored.Count; - Stats.PeersInQueue = peersForDispatch.Count; - Stats.PeersConnecting = bstp.ShortRun; - Stats.PeersConnected = bstp.LongRun; - Stats.PeersChoked = 0; - Stats.PeersUnChoked = 0; - Stats.PeersDownloading = 0; + // Stats & Clean-up + Stats.PeersTotal = peersStored.Count; + Stats.PeersInQueue = peersForDispatch.Count; + Stats.PeersConnecting = bstp.ShortRun; + Stats.PeersConnected = bstp.LongRun; + Stats.PeersChoked = 0; + Stats.PeersUnChoked = 0; + Stats.PeersDownloading = 0; - Stats.PeersDropped = 0; // NOT SET - Stats.PeersFailed1 = 0; // NOT SET - Stats.PeersFailed2 = 0; // NOT SET + Stats.PeersDropped = 0; // NOT SET + Stats.PeersFailed1 = 0; // NOT SET + Stats.PeersFailed2 = 0; // NOT SET - foreach (var thread in bstp.Threads) - { - if (thread.isLongRun) - { - if (thread.peer.status == Peer.Status.READY) - { - if (thread.peer.stageYou.unchoked) - Stats.PeersUnChoked++; - else - Stats.PeersChoked++; - } - else if (thread.peer.status == Peer.Status.DOWNLOADING) + lock (peersForDispatch) + foreach (var thread in bstp.Threads) { - Stats.PeersDownloading++; + if (thread.isLongRun && thread.peer != null) + { + if (thread.peer.status == Peer.Status.READY) + { + if (thread.peer.stageYou.unchoked) + Stats.PeersUnChoked++; + else + Stats.PeersChoked++; + } + else if (thread.peer.status == Peer.Status.DOWNLOADING) + { + Stats.PeersDownloading++; + } + } } - } - } - // Stats -> UI - StatsUpdated?.Invoke(this, new StatsUpdatedArgs(Stats)); + // Stats -> UI + StatsUpdated?.Invoke(this, new StatsUpdatedArgs(Stats)); - // Stats -> Log - if (Options.LogStats) Log("Dumping Stats\r\n" + DumpStats()); - //{ - - // Log($"[STATS] [INQUEUE: {String.Format("{0,3}",Stats.PeersInQueue)}]\t[DROPPED: {String.Format("{0,3}",Stats.PeersDropped)}]\t[CONNECTING: {String.Format("{0,3}",Stats.PeersConnecting)}]\t[FAIL1: {String.Format("{0,3}",Stats.PeersFailed1)}]\t[FAIL2: {String.Format("{0,3}",Stats.PeersFailed2)}]\t[READY: {String.Format("{0,3}",Stats.PeersConnected)}]\t[CHOKED: {String.Format("{0,3}",Stats.PeersChoked)}]\t[UNCHOKED: {String.Format("{0,3}",Stats.PeersUnChoked)}]\t[DOWNLOADING: {String.Format("{0,3}",Stats.PeersDownloading)}]"); - // Log($"[STATS] [CUR MAX: {String.Format("{0:n0}", (Stats.MaxRate / 1024)) + " KB/s"}]\t[DOWN CUR: {String.Format("{0:n0}", (Stats.DownRate / 1024)) + " KB/s"}]\t[DOWN AVG: {String.Format("{0:n0}", (Stats.AvgRate / 1024)) + " KB/s"}]\t[ETA CUR: {TimeSpan.FromSeconds(Stats.ETA).ToString(@"hh\:mm\:ss")}]\t[ETA AVG: {TimeSpan.FromSeconds(Stats.AvgETA).ToString(@"hh\:mm\:ss")}]\t[ETA R: {TimeSpan.FromSeconds((Stats.ETA + Stats.AvgETA)/2).ToString(@"hh\:mm\:ss")}]"); + // Stats -> Log + if (Options.LogStats) Log("Dumping Stats\r\n" + DumpStats()); - // Log($"[STATS] [PIECE TIMEOUTS: {String.Format("{0,4}",Stats.PieceTimeouts)}]\t[HAND TIMEOUTS: {String.Format("{0,4}",Stats.HandshakeTimeouts)}]\t[ALREADYRECV: {String.Format("{0,3}",Stats.AlreadyReceived)}]\t[REJECTED: {String.Format("{0,3}",Stats.Rejects)}]\t[SHA1FAILS:{String.Format("{0,3}",sha1Fails)}]\t[DROPPED BYTES: {Utils.BytesToReadableString(Stats.BytesDropped)}]\t[DHT: {dht?.status}]\t[DHTPEERS: {dhtPeers}]\t[TRACKERSPEERS: {trackersPeers}]\t[SLEEPMODE: {Stats.SleepMode}]"); - // Log($"[STATS] [PROGRESS PIECES: {torrent.data.progress.setsCounter}/{torrent.data.progress.size} | REQ: {torrent.data.requests.setsCounter}]\t[PROGRESS BYTES: {Stats.BytesDownloaded}/{torrent.data.totalSize}]\t[Pieces/Blocks: {torrent.data.pieces}/{torrent.data.blocks}]\t[Piece/Block Length: {torrent.data.pieceSize}|{torrent.data.totalSize % torrent.data.pieceSize}/{torrent.data.blockSize}|{torrent.data.blockLastSize}][Working Pieces: {torrent.data.pieceProgress.Count}]"); - //} + } catch (Exception e) { Log($"[CRITICAL ERROR] Msg: {e.Message}\r\n{e.StackTrace}"); } } public string DumpTorrent() { @@ -825,22 +803,22 @@ public string DumpStats() string includedBytes = $" ({Utils.BytesToReadableString(Stats.BytesCurDownloaded)} / {Utils.BytesToReadableString(Stats.PiecesIncluded * torrent.data.pieceSize)})"; // Not the actual file sizes but pieces size (- first/last chunks) stats += "\n"; stats += $"BitSwarm " + - $"{PadStats(TimeSpan.FromSeconds(curSecond).ToString(@"hh\:mm\:ss"), 13)} | " + + $"{PadStats(TimeSpan.FromSeconds(curSecond).ToString(@"hh\:mm\:ss"), 15)} | " + $"{PadStats("ETA " + TimeSpan.FromSeconds(Stats.AvgETA).ToString(@"hh\:mm\:ss"), 20)} | " + - $"{Utils.BytesToReadableString(Stats.BytesDownloaded)} / {Utils.BytesToReadableString(torrent.data.totalSize)}{(Stats.PiecesIncluded == torrent.data.pieces ? "" : includedBytes)}"; + $"{Utils.BytesToReadableString(Stats.BytesDownloaded)} / {Utils.BytesToReadableString(torrent.data.totalSize)}{(Stats.PiecesIncluded == torrent.data.pieces ? "" : includedBytes)} "; stats += "\n"; - stats += $" v2.2.6 " + - $"{PadStats(String.Format("{0:n0}", Stats.DownRate/1024), 9)} KB/s | " + + stats += $" v2.2.7 " + + $"{PadStats(String.Format("{0:n0}", Stats.DownRate/1024), 11)} KB/s | " + $"{PadStats(String.Format("{0:n1}", ((Stats.DownRate * 8)/1000.0)/1000.0), 15)} Mbps | " + - $"Max: {String.Format("{0:n0}", Stats.MaxRate/1024)} KB/s, {String.Format("{0:n0}", ((Stats.MaxRate * 8)/1000.0)/1000.0)} Mbps"; + $"Max: {String.Format("{0:n0}", Stats.MaxRate/1024)} KB/s, {String.Format("{0:n0}", ((Stats.MaxRate * 8)/1000.0)/1000.0)} Mbps "; stats += "\n"; stats += $" " + - $"{PadStats($"Mode: {mode}", 13)} | " + + $"{PadStats($"Mode: {mode}", 15)} | " + $"{PadStats(" ", 20)} | " + - $"Avg: {String.Format("{0:n0}", Stats.AvgRate/1024)} KB/s, {String.Format("{0:n0}", ((Stats.AvgRate * 8)/1000.0)/1000.0)} Mbps"; + $"Avg: {String.Format("{0:n0}", Stats.AvgRate/1024)} KB/s, {String.Format("{0:n0}", ((Stats.AvgRate * 8)/1000.0)/1000.0)} Mbps "; int progressLen = Stats.Progress.ToString().Length; stats += "\n"; @@ -852,22 +830,22 @@ public string DumpStats() stats += "\n"; stats += $"[PEERS ] " + - $"{PadStats($"{Stats.PeersDownloading}/{Stats.PeersChoked}", 9)} D/W | " + + $"{PadStats($"{Stats.PeersDownloading}/{Stats.PeersChoked}", 11)} D/W | " + $"{PadStats($"{Stats.PeersConnecting}/{Stats.PeersInQueue}/{Stats.PeersTotal}", 14)} C/Q/T | " + - $"{trackersPeers}/{dhtPeers} TRK/DHT {(dht.status == DHT.Status.RUNNING ? "(On)" : "(Off)")}"; + $"{pexPeers}/{trackersPeers}/{dhtPeers} PEX/TRK/DHT {(dht.status == DHT.Status.RUNNING ? "(On)" : "(Off)")} "; stats += "\n"; stats += $"[PIECES] " + - $"{PadStats($"{torrent.data.progress.setsCounter}/{torrent.data.pieces}", 9)} D/T | " + + $"{PadStats($"{torrent.data.progress.setsCounter}/{torrent.data.pieces}", 11)} D/T | " + $"{PadStats($"{torrent.data.requests.setsCounter}", 14)} REQ | " + - $"{Utils.BytesToReadableString(Stats.BytesDropped)} / {Stats.AlreadyReceived} BLK (Drops)"; + $"{Utils.BytesToReadableString(Stats.BytesDropped)} / {Stats.AlreadyReceived} BLK (Drops) "; stats += "\n"; stats += $"[ERRORS] " + - $"{PadStats($"{Stats.PieceTimeouts}", 9)} TMO | " + + $"{PadStats($"{Stats.PieceTimeouts}", 11)} TMO | " + $"{PadStats($"{Stats.Rejects}", 14)} RJS | " + - $"{sha1Fails} SHA"; + $"{sha1Fails} SHA "; stats += "\n"; for (int i=0; i<100; i++) stats += "="; @@ -1077,7 +1055,7 @@ private void Beggar() Log("[BEGGAR ] " + status); } catch (ThreadAbortException) { - } catch (Exception e) { Log($"[BEGGAR] Beggar(), Msg: {e.Message}\r\n{e.StackTrace}"); StatusChanged?.Invoke(this, new StatusChangedArgs(2, e.Message + "\r\n"+ e.StackTrace)); } + } catch (Exception e) { Log($"[CRITICAL ERROR] Msg: {e.Message}\r\n{e.StackTrace}"); StatusChanged?.Invoke(this, new StatusChangedArgs(2, e.Message + "\r\n"+ e.StackTrace)); } if (Utils.IsWindows) Utils.TimeEndPeriod(5); } @@ -1099,14 +1077,14 @@ internal void RequestPiece(Peer peer, bool imBeggar = false) if (peer.stageYou.metadataRequested) return; // Ugly Prison TBR | Start Game with Unknown Bitfield Size So they can start and not be prisoned here - if (peer.stageYou.metadataRequested || peer.stageYou.extensions.ut_metadata == 0) + if (peer.stageYou.extensions.ut_metadata == 0) { while(isRunning && !torrent.metadata.isDone) Thread.Sleep(25); return; } // Ugly Prison TBR | Start Game with Unknown Bitfield Size So they can start and not be prisoned here - if (torrent.metadata.parallelRequests < 1) + else if (torrent.metadata.parallelRequests < 1) { while(isRunning && !torrent.metadata.isDone && torrent.metadata.parallelRequests < 1) Thread.Sleep(25); RequestPiece(peer); diff --git a/BitSwarm/BitSwarm.csproj b/BitSwarm/BitSwarm.csproj index e5ec50a..f7dee34 100644 --- a/BitSwarm/BitSwarm.csproj +++ b/BitSwarm/BitSwarm.csproj @@ -14,12 +14,14 @@ git bitswarm bittorrent torrent client streaming dht © SuRGeoNix 2020 - 2.2.6 - Implemented Ban & Block Diff to identify responsible peers for failure - -Fixed a cirtical (but rare) issue with "broken" peers that were sending invalid data which could cause SHA-1 to fail infinite (re-requesting and failing for the same piece). - 2.2.6.0 - 2.2.6.0 + 2.2.7 + - Adding support for IPv4 & IPv6 Peer Exchange (PEX) / BEP 11 +- Fixing critical but rare issue with null reference and missing lock in FillStats (was crashing the application) +- Fixing critical issue with metadata extension (was listening for receive on remote extension id instead of local) +- Fixing DHT/Tracker sockets issue by ensuring that will listen on IPv4 interface (until IPv6 will be added) +- Fixing a minor issue with DumpStats (was not clearing the line buffer before printing the new one) + 2.2.7.0 + 2.2.7.0 diff --git a/BitSwarm/DHT.cs b/BitSwarm/DHT.cs index e4dee48..c35878a 100644 --- a/BitSwarm/DHT.cs +++ b/BitSwarm/DHT.cs @@ -3,9 +3,10 @@ * * TODO * + * DHT IPv6 Extension http://bittorrent.org/beps/bep_0032.html | We can use the ipv4 dht to get ipv6 nodes (by adding 'want' parameter to the request with both n4 and n6 nodes) but we cant use it to get 18-octet IPv6 values + * * 1. Min ThreadPool for Nodes to avoid re-creating threads all the time * 2. Possible review the new rEP / ipEP to ensure not receiving wrong packets (maybe rEP for all threads and also ipEP on thread 0) - * */ using System; @@ -117,8 +118,9 @@ public DHT(string infoHash, Options? opt = null) for (int i=0; i<20; i++) infoHashBits[i] = Convert.ToString(infoHashBytes[i], 2).PadLeft(8, '0'); - udpClient = new UdpClient(0); - ipEP = (IPEndPoint) udpClient.Client.LocalEndPoint; //new IPEndPoint(IPAddress.Any, 0); + udpClient = new UdpClient(0, AddressFamily.InterNetwork); // Ensure that we use IPv4 DHT + ipEP = (IPEndPoint) udpClient.Client.LocalEndPoint; + udpClient.Client.SendTimeout = options.ConnectionTimeout; udpClient.Client.ReceiveTimeout = options.ConnectionTimeout; udpClient.Client.Ttl = 255; @@ -371,14 +373,14 @@ private void GetPeers(Node node, int selfRecursionLevel = 0) string curIP = (new IPAddress(Utils.ArraySub(ref value, 0, 4))).ToString(); UInt16 curPort = (UInt16) BitConverter.ToInt16(Utils.ArraySub(ref value, 4, 2, true), 0); - if (curPort < 100) continue; // Drop fake + if (curPort < 500) continue; // Drop fake / Avoid DDOS //if (options.Verbosity > 0) Log($"[{node.distance}] [{node.host}] [PEER] {curIP}:{curPort}"); curPeers[curIP] = curPort; } - if (curPeers.Count > 0) options.Beggar.FillPeers(curPeers, BitSwarm.PeersStorage.DHTNEW); + if (curPeers.Count > 0) options.Beggar.FillPeers(curPeers, BitSwarm.PeersStorage.DHT); //if (options.Verbosity > 0) Log($"[{node.distance}] [{node.host}] [NEW PEERS] {newPeers}"); diff --git a/BitSwarm/Peer.cs b/BitSwarm/Peer.cs index a1a1377..e1d5cf6 100644 --- a/BitSwarm/Peer.cs +++ b/BitSwarm/Peer.cs @@ -25,10 +25,11 @@ internal class Peer public static readonly byte[] BIT_PROTO = Utils.ArrayMerge(new byte[] {0x13}, Encoding.UTF8.GetBytes("BitTorrent protocol")); public static readonly byte[] EXT_PROTO = Utils.ArrayMerge(new byte[] {0, 0, 0, 0}, new byte[] {0 , 0x10, 0, (0x1 | 0x4)}); - // [HANDSHAKE EXTENDED] | http://bittorrent.org/beps/bep_0010.html | m-> {"key", "value"}, p, v, yourip, ipv6, ipv4, reqq | Static EXT_BDIC - public static readonly byte[] EXT_BDIC = (new BDictionary{ {"e", 0 }, {"m" , new BDictionary{ {"ut_metadata", 2 } /*,{"ut_pex" , 1 }*/ } }, { "reqq" , 250 } }).EncodeAsBytes(); public static readonly int MAX_DATA_SIZE = 0x4000; + // [HANDSHAKE EXTENDED] | http://bittorrent.org/beps/bep_0010.html | m-> {"key", "value"}, p, v, yourip, ipv6, ipv4, reqq | Static EXT_BDIC + public static byte[] EXT_BDIC; // Will be set by bitswarm's setup + public static byte[] HANDSHAKE_BYTES; // Will be set by bitswarm's setup public static BitSwarm Beggar; // Will be change later on for multiple-instances @@ -71,18 +72,20 @@ public static class Messages // LTEP Handshake public const byte EXTENDED_HANDSHAKE = 0x00; + // [msg-id = m -> ut_pex] + // Peer Exchange (PEX) | http://bittorrent.org/beps/bep_0011.html + public const byte EXT_UT_PEX = 0x01; + // [msg-id = m -> ut_metadata] // Metadata Extenstion | http://www.bittorrent.org/beps/bep_0009.html | Extension for Peers to Send Metadata Files // Bencoded [] + public const byte EXT_UT_METADATA = 0x02; public const byte METADATA_REQUEST = 0x00; public const byte METADATA_RESPONSE = 0x01; public const byte METADATA_REJECT = 0x02; - // [msg-id = m -> ut_pex] - // Peer Exchange (PEX) | http://bittorrent.org/beps/bep_0011.html - // [msg-id = m -> lt_donthave] - // DontHave | http://bittorrent.org/beps/bep_0054.html | Old alternative of Fast Extension + // DontHave | http://bittorrent.org/beps/bep_0054.html | Extension for advertising that it no longer has a piece (on previous Have/BitField messages) } /* Main Implementation | Run() @@ -154,11 +157,14 @@ public class Stage public struct Extensions { public byte ut_metadata; + //public byte ut_pex; } public string host { get; private set; } public int port { get; private set; } + //public byte[] remotePeerID { get; private set;} + public Status status; public Stage stageYou; @@ -251,6 +257,7 @@ public bool SendHandshake() if (Beggar.Options.Verbosity > 0) Log(3, "[HAND] Sending"); tcpStream.Write(HANDSHAKE_BYTES, 0, HANDSHAKE_BYTES.Length); Receive(BIT_PROTO.Length + EXT_PROTO.Length + 20 + 20); + //remotePeerID = Utils.ArraySub(ref recvBuff, 47, 20); if (Beggar.Options.Verbosity > 0) Log(3, "[HAND] Received"); lastPieces = new List>(); @@ -495,12 +502,7 @@ private void ProcessMessage() { status = Status.READY; - if (stageYou.unchoked) - Beggar.RequestPiece(this); - - // Will cause infinite re-requesting - //else if (allowFastPieces.Count > 0) - //Beggar.RequestFastPiece(this); + if (stageYou.unchoked) Beggar.RequestPiece(this); } return; @@ -542,6 +544,8 @@ private void ProcessMessage() case Messages.EXTENDED: Receive(1); // MSG Extension Id + if (Beggar.Options.Verbosity > 0) Log(3, "[MSG ] Extended ..."); + if (recvBuff[0] == Messages.EXTENDED_HANDSHAKE) { if (Beggar.Options.Verbosity > 0) Log(3, "[HAND] Extended Received"); @@ -554,6 +558,8 @@ private void ProcessMessage() if (cur != null) stageYou.extensions.ut_metadata = (byte) ((int) cur); cur = Utils.GetFromBDic(extDic, new string[] {"m", "ut_metadata"}); if (cur != null) stageYou.extensions.ut_metadata = (byte) ((int) cur); + //cur = Utils.GetFromBDic(extDic, new string[] {"m", "ut_pex"}); + //if (cur != null) stageYou.extensions.ut_pex = (byte) ((int) cur); // MSG Extended Handshake | Reply SendExtendedHandshake(); @@ -563,10 +569,60 @@ private void ProcessMessage() return; } - // TODO: recvBuff[0] == extensions.ut_pex | PEX http://bittorrent.org/beps/bep_0011.html + // Peer Exchange (PEX) | http://bittorrent.org/beps/bep_0011.html + else if (recvBuff[0] == Messages.EXT_UT_PEX) + { + /* TODO: + * + * By adding IPv6 we loose uniquness by host on peers, we should change the uniquness based on remotePeerId (currently possible we connect to the same IPv4 / IPv6 peer ?) - We can also get ipv6 from Extended message (->ipv6) + * Possible process also dropped to remove peers from main storage (or even "ban" them to avoid re-push them in the queue) + */ + + if (Beggar.Options.Verbosity > 0) Log(3, "[PEX] ..."); + + Receive(msgLen - 2); + + byte[] buff = msgLen - 2 == MAX_DATA_SIZE ? recvBuffMax : recvBuff; + + BDictionary extDic = bParser.Parse(recvBuff); + byte[] buffAdded = new byte[0]; + Dictionary peers = new Dictionary(); + + if (extDic.ContainsKey("added")) buffAdded = ((BString)extDic["added"]).Value.ToArray(); + + for (int i = 0; i < buffAdded.Length / 6; i++) + { + System.Net.IPAddress curIP = new System.Net.IPAddress(Utils.ArraySub(ref buffAdded, (uint)i * 6, 4, false)); + UInt16 curPort = (UInt16)BitConverter.ToInt16(Utils.ArraySub(ref buffAdded, (uint)4 + (i * 6), 2, true), 0); + + if (curPort < 500) continue; // Drop fake / Avoid DDOS + + peers[curIP.ToString()] = curPort; + } + + buffAdded = new byte[0]; + if (extDic.ContainsKey("added6")) buffAdded = ((BString) extDic["added6"]).Value.ToArray(); + + for (int i=0; i 0) Beggar.FillPeers(peers, BitSwarm.PeersStorage.PEX); + + //if (Beggar.Options.Verbosity > 0) Log(3, $"[PEX] {peers.Count}"); + + return; + } + // Extension for Peers to Send Metadata Files | info-dictionary part of the .torrent file | http://bittorrent.org/beps/bep_0009.html - else if (recvBuff[0] == stageYou.extensions.ut_metadata && stageYou.extensions.ut_metadata != 0) + else if (recvBuff[0] == Messages.EXT_UT_METADATA) { // MSG Extended Metadata if (Beggar.Options.Verbosity > 0) Log(3, "[META] ..."); diff --git a/BitSwarm/Torrent.cs b/BitSwarm/Torrent.cs index b7141b9..7ed3d4e 100644 --- a/BitSwarm/Torrent.cs +++ b/BitSwarm/Torrent.cs @@ -52,7 +52,7 @@ public struct TorrentData public int pieces { get; set; } public int pieceSize { get; set; } - public int pieceLastSize { get; set; } + public int pieceLastSize { get; set; } // NOTE: it can be 0, it should be equals with pieceSize in case of totalSize % pieceSize = 0 public int blocks { get; set; } public int blockSize { get; set; } @@ -209,7 +209,7 @@ public void FillFromInfo(BDictionary bInfo) data.pieces = file.pieces.Count; data.pieceSize = file.pieceLength; - data.pieceLastSize = (int) (data.totalSize % data.pieceSize); + data.pieceLastSize = (int) (data.totalSize % data.pieceSize); // NOTE: it can be 0, it should be equals with pieceSize in case of totalSize % pieceSize = 0 data.progress = new BitField(data.pieces); data.requests = new BitField(data.pieces); diff --git a/BitSwarm/Tracker.cs b/BitSwarm/Tracker.cs index eaafa45..f5b65cd 100644 --- a/BitSwarm/Tracker.cs +++ b/BitSwarm/Tracker.cs @@ -117,14 +117,15 @@ private void InitializeUDP() if (ip.AddressFamily == AddressFamily.InterNetwork) rEP = new IPEndPoint(ip, port); - if (rEP == null) - foreach (var ip in ips) - if (ip.AddressFamily == AddressFamily.InterNetworkV6) - { rEP = new IPEndPoint(ip, port); break; } + // Need to implement also IPv6 retrieval to support IPv6 + //if (rEP == null) + // foreach (var ip in ips) + // if (ip.AddressFamily == AddressFamily.InterNetworkV6) + // { rEP = new IPEndPoint(ip, port); break; } if (rEP == null) { Log("DNS failed"); failed = true; return; } - udpClient = new UdpClient(0); + udpClient = new UdpClient(0, AddressFamily.InterNetwork); // Currently only IPv4 udpClient.Client.ReceiveTimeout = options.ReceiveTimeout; } @@ -239,7 +240,7 @@ public bool AnnounceTCP(Int32 num_want = -1, Int64 downloaded = 0, Int64 left = if (options.Verbosity > 0) Log($"Success ({peers.Count} Peers)"); - if (peers.Count > 0) Beggar.FillPeers(peers, BitSwarm.PeersStorage.TRACKERSNEW); + if (peers.Count > 0) Beggar.FillPeers(peers, BitSwarm.PeersStorage.TRACKER); } catch (Exception e) { @@ -337,12 +338,15 @@ private void ReceiveUDP(IAsyncResult ar) { IPAddress curIP = new IPAddress(Utils.ArraySub(ref recvBuff,(uint) (20 + (i*6)), 4, false)); UInt16 curPort = (UInt16) BitConverter.ToInt16(Utils.ArraySub(ref recvBuff,(uint) (24 + (i*6)), 2, true), 0); - if (curPort > 0) peers[curIP.ToString()] = curPort; + + if (curPort < 500) continue; // Drop fake / Avoid DDOS + + peers[curIP.ToString()] = curPort; } if (options.Verbosity > 0) Log($"Success ({peers.Count} Peers)"); - if (peers.Count > 0) Beggar.FillPeers(peers, BitSwarm.PeersStorage.TRACKERSNEW); + if (peers.Count > 0) Beggar.FillPeers(peers, BitSwarm.PeersStorage.TRACKER); // Check with bytes cause Peers maybe < 200 | We drop some invalid ports or we dont read them properly? if (recvBuff.Length == 1220) { curRecursions++; AnnounceUDP(); }