-
Notifications
You must be signed in to change notification settings - Fork 26
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Code now targets .NET 6. Documents loaded from SEC URIs are now retri…
…eved using special handling compatible with the SEC usage rules.
- Loading branch information
1 parent
ee3c659
commit a85bbb2
Showing
24 changed files
with
190 additions
and
64 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,7 +1,10 @@ | ||
| using JeffFerguson.Gepsio.IoC; | ||
| using JeffFerguson.Gepsio.Xml.Interfaces; | ||
| using System; | ||
| using System.Collections.Generic; | ||
| using System.IO; | ||
| using System.Net; | ||
| using System.Net.Http; | ||
| using System.Threading.Tasks; | ||
|
|
||
| namespace JeffFerguson.Gepsio | ||
|
|
@@ -174,13 +177,85 @@ public XbrlDocument() | |
| /// </param> | ||
| public void Load(string Filename) | ||
| { | ||
| var SchemaValidXbrl = Container.Resolve<IDocument>(); | ||
| SchemaValidXbrl.Load(Filename); | ||
| this.Filename = Filename; | ||
| this.Path = System.IO.Path.GetDirectoryName(this.Filename); | ||
| Parse(SchemaValidXbrl); | ||
| if (IsSecUri(Filename) == true) | ||
| { | ||
| LoadFromSec(Filename); | ||
| } | ||
| else | ||
| { | ||
| var SchemaValidXbrl = Container.Resolve<IDocument>(); | ||
| SchemaValidXbrl.Load(Filename); | ||
| this.Filename = Filename; | ||
| this.Path = System.IO.Path.GetDirectoryName(this.Filename); | ||
| Parse(SchemaValidXbrl); | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Synchronously load a document directly from the SEC Web site. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// The SEC Web site does not allow code to scrape documents from the site | ||
| /// without supplying appropriate HTTP headers. Without the correct HTTP | ||
| /// headers, simply calling XDocument.Load() for a document stored at the | ||
| /// SEC Web site will fail, most likely with an HTTP 403 error code. Since | ||
| /// Gepsio contains unit tests that reference documents stored at the SEC Web | ||
| /// site, support for SEC-compatible HTTP headers is necessary. See documentation | ||
| /// at https://www.sec.gov/os/accessing-edgar-data for more information. | ||
| /// </remarks> | ||
| /// <param name="path"> | ||
| /// The path of the document to load from the SEC Web site. | ||
| /// </param> | ||
| private void LoadFromSec(string path) | ||
| { | ||
| using(var request = new HttpRequestMessage(HttpMethod.Get, path)) | ||
| { | ||
| var httpClientHandler = new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }; | ||
| using (var httpClient = new HttpClient(httpClientHandler)) | ||
| { | ||
| request.Headers.TryAddWithoutValidation("User-Agent", "Gepsio [email protected]"); | ||
| request.Headers.Add("Accept-Encoding", "gzip, deflate"); | ||
| request.Headers.Add("Host", "www.sec.gov"); | ||
| var response = httpClient.Send(request); | ||
| response.EnsureSuccessStatusCode(); | ||
| var responseAsStream = response.Content.ReadAsStream(); | ||
| Load(responseAsStream); | ||
| } | ||
| } | ||
| } | ||
| /// <summary> | ||
| /// Asynchronously load a document directly from the SEC Web site. | ||
| /// </summary> | ||
| /// <remarks> | ||
| /// The SEC Web site does not allow code to scrape documents from the site | ||
| /// without supplying appropriate HTTP headers. Without the correct HTTP | ||
| /// headers, simply calling XDocument.Load() for a document stored at the | ||
| /// SEC Web site will fail, most likely with an HTTP 403 error code. Since | ||
| /// Gepsio contains unit tests that reference documents stored at the SEC Web | ||
| /// site, support for SEC-compatible HTTP headers is necessary. See documentation | ||
| /// at https://www.sec.gov/os/accessing-edgar-data for more information. | ||
| /// </remarks> | ||
| /// <param name="path"> | ||
| /// The path of the document to load from the SEC Web site. | ||
| /// </param> | ||
| private async Task LoadFromSecAsync(string path) | ||
| { | ||
| using(var request = new HttpRequestMessage(HttpMethod.Get, path)) | ||
| { | ||
| var httpClientHandler = new HttpClientHandler() { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }; | ||
| using (var httpClient = new HttpClient(httpClientHandler)) | ||
| { | ||
| request.Headers.TryAddWithoutValidation("User-Agent", "Gepsio [email protected]"); | ||
| request.Headers.Add("Accept-Encoding", "gzip, deflate"); | ||
| request.Headers.Add("Host", "www.sec.gov"); | ||
| var response = await httpClient.SendAsync(request); | ||
| response.EnsureSuccessStatusCode(); | ||
| var responseAsStream = await response.Content.ReadAsStreamAsync(); | ||
| await LoadAsync(responseAsStream); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Asynchronously loads a local filesystem or Internet-accessible XBRL document containing | ||
| /// XBRL data. | ||
|
|
@@ -200,11 +275,23 @@ public void Load(string Filename) | |
| /// </param> | ||
| public async Task LoadAsync(string Filename) | ||
| { | ||
| var SchemaValidXbrl = Container.Resolve<IDocument>(); | ||
| await SchemaValidXbrl.LoadAsync(Filename); | ||
| this.Filename = Filename; | ||
| this.Path = System.IO.Path.GetDirectoryName(this.Filename); | ||
| Parse(SchemaValidXbrl); | ||
| //var SchemaValidXbrl = Container.Resolve<IDocument>(); | ||
| //await SchemaValidXbrl.LoadAsync(Filename); | ||
| //this.Filename = Filename; | ||
| //this.Path = System.IO.Path.GetDirectoryName(this.Filename); | ||
| //Parse(SchemaValidXbrl); | ||
| if (IsSecUri(Filename) == true) | ||
| { | ||
| await LoadFromSecAsync(Filename); | ||
| } | ||
| else | ||
| { | ||
| var SchemaValidXbrl = Container.Resolve<IDocument>(); | ||
| await SchemaValidXbrl.LoadAsync(Filename); | ||
| this.Filename = Filename; | ||
| this.Path = System.IO.Path.GetDirectoryName(this.Filename); | ||
| Parse(SchemaValidXbrl); | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
|
|
@@ -373,6 +460,28 @@ public async Task LoadAsync(Stream dataStream) | |
| Parse(SchemaValidXbrl); | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Determines whether or not a URI references the SEC Web site. | ||
| /// </summary> | ||
| /// <param name="uriPath"> | ||
| /// The URI to check. | ||
| /// </param> | ||
| /// <returns> | ||
| /// True if the supplied URI is an SEC URI; false otherwise. | ||
| /// </returns> | ||
| private bool IsSecUri(string uriPath) | ||
| { | ||
| try | ||
| { | ||
| var uriToInspect = new Uri(uriPath); | ||
| return uriToInspect.Host.ToLower().Trim().Equals("www.sec.gov"); | ||
| } | ||
| catch(UriFormatException) | ||
| { | ||
| return false; | ||
| } | ||
| } | ||
|
|
||
| /// <summary> | ||
| /// Parse the document, looking for fragments that can be processed. | ||
| /// </summary> | ||
|
|
||
Oops, something went wrong.