Merge pull request #8 from luizmb/master

Fix SSH repos and repos with no LICENSE

Sources/Models/Bridges.swift

@@ -11,7 +11,7 @@ public typealias PackageLicense = (package: ResolvedPackage, license: GitHubLice
 public func extractPackageGitHubRepositories(from spmFile: ResolvedPackageContent) -> [PackageRepository] {
     spmFile.object.pins.compactMap { spmPackage in
         guard let repository = githubRepository(from: spmPackage.repositoryURL).value else {
-            print("Ignoring project \(spmPackage.package) because we don't know how to fetch the license from it")
+            print("⚠️ Ignoring project \(spmPackage.package) because we don't know how to fetch the license from it")
             return nil
         }
 
@@ -26,16 +26,38 @@ public func fetchGithubLicenses(
 ) -> Reader<(Request, Decoder<GitHubLicense>), Publishers.Promise<[PackageLicense], GeneratePlistError>> {
     Reader { requester, decoder in
         Publishers.Promise.zip(
-            packageRepositories.map { packageRepository in
+            packageRepositories.map { packageRepository -> Publishers.Promise<PackageLicense?, GeneratePlistError> in
                 githubLicensingAPI(
                     repository: packageRepository.repository,
                     githubClientID: githubClientID,
                     githubClientSecret: githubClientSecret
                 )
                 .inject((requester, decoder))
                 .map { license in PackageLicense(package: packageRepository.package, license: license) }
+                .catch { error in
+                    // Some special errors we allow to go through, so this specific package will be removed
+                    // from the final plist, but the others will be there. Example of that: when one repo doesn't
+                    // have the LICENSES file (404), we just log to the console a message similar to the one for
+                    // wrong URLs.
+                    //
+                    // Other cases, such as 403 for example, we want to interrupt the process completely, because
+                    // we exceeded the Github API budget and none of the licenses from now on will succeed.
+                    //
+                    // These are two different levels of errors.
+
+                    if case .githubLicenseNotFound = error {
+                        print("⚠️ Ignoring project \(packageRepository.package.package) becuse we failed to download the LICENSE from github. Error: \(error)")
+
+                        // Return a nil PackageLicense for now, we filter them out in the last step (outside the Promise.zip)
+                        return .init(value: nil)
+                    }
+                    return .init(error: error)
+                }
             }
-        )
+        ).map { arrayOfOptionalPackages in
+            // Remove nil packages, that felt in the case of "soft" error (see comments above)
+            arrayOfOptionalPackages.compactMap(identity)
+        }
     }
 }
 

Sources/Models/GeneratePlistError.swift

@@ -11,6 +11,8 @@ public enum GeneratePlistError: Error {
     case invalidLicenseMetadataURL
     case unknownRepository
     case githubAPIURLError(URLError)
+    case githubLicenseNotFound
+    case githubAPIBudgetExceeded
     case githubAPIInvalidResponse(URLResponse)
     case githubLicenseJsonCannotBeDecoded(url: URL, json: String?, error: Error)
     case githubLicenseCannotBeDownloaded(URL)

Sources/Models/GitHub.swift

@@ -34,22 +34,32 @@ public struct GitHubLicense: Decodable {
     }
 }
 
-func githubRepository(from url: URL) -> Result<GitHubRepository, GeneratePlistError> {
-    let gitDomain = "github.com"
-    let gitSuffix = ".git"
+extension String {
+    func replacingOccurrences(of strings: [String], with newString: String) -> String {
+        strings.reduce(self) { partialResult, itemToReplace in
+            partialResult.replacingOccurrences(of: itemToReplace, with: newString)
+        }
+    }
+}
 
-    guard let host = url.host,
-        host.contains(gitDomain),
-        url.pathComponents.count >= 2,
-        let lastPathComponent = url.pathComponents.last
+func githubRepository(from url: URL) -> Result<GitHubRepository, GeneratePlistError> {
+    let cleanupList = [
+        "[email protected]:",
+        "https://github.com/",
+        "https://www.github.com/",
+        "http://github.com/",
+        "http://www.github.com/",
+        ".git"
+    ]
+
+    let parts = url.absoluteString
+        .replacingOccurrences(of: cleanupList, with: "")
+        .split(separator: "/")
+
+    guard
+        let owner = parts[safe: 0].map(String.init),
+        let name = parts[safe: 1].map(String.init)
     else { return .failure(.unknownRepository) }
-
-    guard let owner = url.pathComponents[safe: 1], !owner.isEmpty else {
-        return .failure(.invalidLicenseMetadataURL)
-    }
-    let name = lastPathComponent.contains(gitSuffix)
-        ? String(lastPathComponent.dropLast(gitSuffix.count))
-        : String(lastPathComponent.replacingOccurrences(of: gitSuffix, with: ""))
 
     return .success(
         GitHubRepository(
@@ -80,10 +90,22 @@ public func githubLicensingAPI(
         requester(request)
             .mapError(GeneratePlistError.githubAPIURLError)
             .flatMapResult { data, response -> Result<Data, GeneratePlistError> in
-                guard let httpResponse = response as? HTTPURLResponse,
-                    200..<300 ~= httpResponse.statusCode else {
-                        return .failure(.githubAPIInvalidResponse(response))
+                guard let httpResponse = response as? HTTPURLResponse else {
+                    return .failure(.githubAPIInvalidResponse(response))
+                }
+
+                guard httpResponse.statusCode != 404 else {
+                    return .failure(.githubLicenseNotFound)
+                }
+
+                guard httpResponse.statusCode != 403 else {
+                    return .failure(.githubAPIBudgetExceeded)
+                }
+
+                guard 200..<300 ~= httpResponse.statusCode else {
+                    return .failure(.githubAPIInvalidResponse(response))
                 }
+
                 return .success(data)
             }
             .flatMapResult { data in

Sources/SwiftPackageAcknowledgement/Commands/GeneratePlist.swift

@@ -50,8 +50,15 @@ struct GeneratePlist: ParsableCommand {
             .sinkBlockingAndExit(
                 receiveCompletion: { completion in
                     switch completion {
-                    case let .failure(error): print("An error has occurred: \(error)")
-                    case .finished:           print("Done!")
+                    case let .failure(error):
+                        print("\n💥 An error has occurred: \(error)")
+                        if case .githubAPIBudgetExceeded = error {
+                            print("Github API has a limit of 60 requests per hour when you don't use a Developer Token.")
+                            print("Please consider going to https://github.com/settings/developers, creating an OAuth App and " +
+                                  "providing Client ID and Client Secret Token in the script call.")
+                        }
+                    case .finished:
+                        print("\n✅ Done!")
                     }
                 },
                 receiveValue: { _ in }

Tests/SwiftPackageAcknowledgementTests/BridgeTests.swift

@@ -16,6 +16,21 @@ class BridgeTests: XCTestCase {
                 package: "Commander",
                 repositoryURL: URL(string: "https://github.com/kylef/Commander")!,
                 state: .init()
+            ),
+            .init(
+                package: "SwiftPackageAcknowledgement",
+                repositoryURL: URL(string: "http://github.com/teufelaudio/SwiftPackageAcknowledgement")!,
+                state: .init()
+            ),
+            .init(
+                package: "FoundationExtensions",
+                repositoryURL: URL(string: "https://www.github.com/teufelaudio/FoundationExtensions")!,
+                state: .init()
+            ),
+            .init(
+                package: "NetworkExtensions",
+                repositoryURL: URL(string: "[email protected]:teufelaudio/NetworkExtensions.git")!,
+                state: .init()
             )
         ]
         let content = ResolvedPackageContent(
@@ -28,12 +43,26 @@ class BridgeTests: XCTestCase {
 
         // assert
         let repositories = packageRepositories.map(\.repository)
+        XCTAssertEqual(repositories.count, 5)
+
         let acknowListRepo = repositories[0]
         XCTAssertEqual(acknowListRepo.name, "AcknowList")
         XCTAssertEqual(acknowListRepo.owner, "vtourraine")
 
         let commanderRepo = repositories[1]
         XCTAssertEqual(commanderRepo.name, "Commander")
         XCTAssertEqual(commanderRepo.owner, "kylef")
+
+        let spmAckRepo = repositories[2]
+        XCTAssertEqual(spmAckRepo.name, "SwiftPackageAcknowledgement")
+        XCTAssertEqual(spmAckRepo.owner, "teufelaudio")
+
+        let foundationExtRepo = repositories[3]
+        XCTAssertEqual(foundationExtRepo.name, "FoundationExtensions")
+        XCTAssertEqual(foundationExtRepo.owner, "teufelaudio")
+
+        let networkExtRepo = repositories[4]
+        XCTAssertEqual(networkExtRepo.name, "NetworkExtensions")
+        XCTAssertEqual(networkExtRepo.owner, "teufelaudio")
     }
 }