Merge branch 'zio:main' into netty-internal-logging-support

.github/workflows/ci.yml

@@ -91,6 +91,12 @@ jobs:
       - name: Build project
         run: sbt '++ ${{ matrix.scala }}' test
 
+      - uses: coursier/setup-action@v1
+
+      - name: Test sbt plugin
+        if: ${{ github.event_name == 'pull_request' }} && matrix.scala == '2.12.19'
+        run: sbt ++2.12.19 zioHttpGenSbt/scripted
+
       - uses: coursier/setup-action@v1
         with:
           apps: sbt
@@ -106,7 +112,7 @@ jobs:
         run: sbt '++ ${{ matrix.scala }}' zioHttpShadedTests/test
 
       - name: Compress target directories
-        run: tar cf targets.tar sbt-zio-http-grpc/target zio-http-cli/target target zio-http/jvm/target zio-http-docs/target sbt-zio-http-grpc-tests/target zio-http-gen/target zio-http-benchmarks/target zio-http-tools/target zio-http-example/target zio-http-testkit/target zio-http/js/target zio-http-htmx/target project/target
+        run: tar cf targets.tar sbt-zio-http-grpc/target zio-http-gen-sbt-plugin/target zio-http-cli/target target zio-http/jvm/target zio-http-docs/target sbt-zio-http-grpc-tests/target zio-http-gen/target zio-http-benchmarks/target zio-http-tools/target zio-http-example/target zio-http-testkit/target zio-http/js/target zio-http-htmx/target project/target
 
       - name: Upload target directories
         uses: actions/upload-artifact@v4

build.sbt

@@ -93,6 +93,22 @@ ThisBuild / githubWorkflowBuildPreamble := Seq(
   ),
 )
 
+ThisBuild / githubWorkflowBuild := {
+  (ThisBuild / githubWorkflowBuild).value ++ WorkflowJob(
+    "testSbtPlugin",
+    "Test sbt plugin",
+    List(
+      WorkflowStep.Use(UseRef.Public("coursier", "setup-action", "v1")),
+      WorkflowStep.Run(
+        name = Some(s"Test sbt plugin"),
+        commands = List(s"sbt ++${Scala212} zioHttpGenSbt/scripted"),
+        cond = Some(s"$${{ github.event_name == 'pull_request' }} && matrix.scala == '$Scala212'"),
+      ),
+    ),
+    scalas = List(Scala212),
+  ).steps
+}
+
 ThisBuild / githubWorkflowBuildPostamble :=
   WorkflowJob(
     "checkDocGeneration",
@@ -350,6 +366,26 @@ lazy val zioHttpGen = (project in file("zio-http-gen"))
   )
   .dependsOn(zioHttpJVM)
 
+lazy val zioHttpGenSbt = (project in file("zio-http-gen-sbt-plugin"))
+  .enablePlugins(SbtPlugin)
+  .settings(publishSetting(true))
+  .settings(
+    name := "zio-http-sbt-codegen",
+    sbtPlugin    := true,
+    scalaVersion := Scala212,
+    semanticdbEnabled := true,
+    semanticdbVersion := scalafixSemanticdb.revision,
+    scalacOptions ++= stdOptions ++ extraOptions(scalaVersion.value),
+    sbtTestDirectory := sourceDirectory.value / "sbt-test",
+    scriptedLaunchOpts += ("-Dplugin.version=" + version.value),
+    scriptedBufferLog := false,
+    libraryDependencies ++= Seq(
+      `zio-json-yaml`,
+      `zio-test`,
+      `zio-test-sbt`,
+    )
+  ).dependsOn(LocalProject("zioHttpGen"))
+
 lazy val sbtZioHttpGrpc = (project in file("sbt-zio-http-grpc"))
   .settings(stdSettings("sbt-zio-http-grpc"))
   .settings(publishSetting(true))

docs/reference/openapi-gen-sbt-plugin.md

@@ -0,0 +1,75 @@
+---
+id: openapi-gen-sbt-plugin
+title: OpenAPI codegen sbt plugin
+---
+
+This plugin allows to easily generate scala source code with zio-http Endpoints from OpenAPI spec files.
+
+## How to use
+
+The plugin offers 2 modes of operation that can be mixed and used together:
+- Generating from unmanaged static OpenAPI spec files
+- Generating from managed dynamic OpenAPI spec files
+
+in `project/plugins.sbt` add the following line:
+```scala
+addSbtPlugin("dev.zio" % "zio-http-sbt-codegen" % "@VERSION@") // make sure the version of the sbt plugin
+                                                               // matches the version of zio-http you are using
+```
+
+in `build.sbt` enable the plugin by adding:
+```scala
+enablePlugins(ZioHttpCodegen)
+```
+
+### 1. Generating from unmanaged static OpenAPI spec files
+Place your manually curated OpenAPI spec files (`.yml`, `.yaml`, or `.json`) in `src/main/oapi/<path as package>/<openapi spec file>`.\
+That's it. No other configuration is needed for basic usage. \
+Once you `compile` your project, the `zioHttpCodegenMake` task is automatically invoked, and the generated code will be placed under `target/scala-<scala_binary_version>/src_managed/main/scala`.
+
+### 2. Generating from managed dynamic OpenAPI spec files
+In this mode, you can hook into `ZIOpenApi / sourceGenerators` a task to generate OpenAPI spec file, exactly like you would do with regular `Compile / sourceGenerators` for scala source files.
+You might have some OpenAPI spec files hosted on [swaggerhub](https://app.swaggerhub.com/) or a similar service, 
+or maybe you use services that expose OpenAPI specs via REST API, or perhaps you have a local project that can build its own spec and you want to run the spec generate command.
+Whatever the scenario you're dealing with, it can be very handy to dynamically fetch/generate the latest most updated spec file, so the generated code stays up to date with any changes introduced.
+
+Here's how you can do it:
+```scala
+import gigahorse.support.apachehttp.Gigahorse
+import scala.concurrent.Await
+import scala.concurrent.duration.DurationInt
+
+ZIOpenApi / sourceGenerators += Def.task[Seq[File]] {
+  // we'll fetch a spec from https://www.petstore.dev/
+  // gigahorse comes builtin with sbt, but any other http client can be used
+  val http = Gigahorse.http(Gigahorse.config)
+  val request = Gigahorse.url("https://raw.githubusercontent.com/readmeio/oas-examples/main/3.0/yaml/response-http-behavior.yaml")
+  val response = http.run(request, Gigahorse.asString)
+  val content = Await.result(response, 1.minute)
+
+  // path under target/scala-<scala_bin_version>/src_managed/oapi/
+  // corresponds to the package where scala sources will be generated
+  val outFile = (ZIOpenApi / sourceManaged).value / "dev" / "petstore" / "http" / "test" / "api.yaml"
+  IO.write(outFile, content)
+
+  // as long the task yields a Seq[File] of valid OpenAPI spec files,
+  // and those files follow the path structure `src_managed/oapi/<path as package>/<openapi spec file>`,
+  // the plugin will pick it up, and generate the corresponding scala sources.
+  Seq(outFile)
+}
+```
+
+## Configuration
+The plugin offers a setting key which you can set to control how code is generated:
+```scala
+zioHttpCodegenConf := zio.http.gen.openapi.Config.default
+```
+
+## Caveats
+The plugin allows you to provide multiple files.
+Note that if you place multiple files in the same directory,
+which means same package for the generated code - you must make sure there are no "collisions" between generated classes.
+If the same class is going to be generated differently in different files, you probably want to have a different package for it.
+
+Also, please note that the plugin relies on the file extension to determine how to parse it.
+So files must have the correct extension (`.yml`, `.yaml`, or `.json`), and the content must be formatted accordingly.

project/BuildHelper.scala

@@ -12,7 +12,7 @@ object BuildHelper extends ScalaSettings {
   val ScoverageVersion = "2.0.12"
   val JmhVersion       = "0.4.7"
 
-  private val stdOptions = Seq(
+  val stdOptions = Seq(
     "-deprecation",
     "-encoding",
     "UTF-8",

project/Dependencies.scala

@@ -2,7 +2,7 @@ import sbt.*
 
 object Dependencies {
   val JwtCoreVersion                = "10.0.1"
-  val NettyVersion                  = "4.1.112.Final"
+  val NettyVersion                  = "4.1.116.Final"
   val NettyIncubatorVersion         = "0.0.25.Final"
   val ScalaCompactCollectionVersion = "2.12.0"
   val ZioVersion                    = "2.1.11"

project/build.properties

@@ -1 +1 @@
-sbt.version=1.10.0
+sbt.version=1.10.6

zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/Format.scala

@@ -0,0 +1,11 @@
+package zio.http.gen.sbt
+
+sealed trait Format
+object Format {
+  case object YAML extends Format
+  case object JSON extends Format
+  def fromFileName(fileName: String): Format =
+    if (fileName.endsWith("yml") || fileName.endsWith("yaml")) YAML
+    else if (fileName.endsWith("json")) JSON
+    else throw new Exception(s"Unknown format for file $fileName")
+}

zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/ZioHttpCodegen.scala

@@ -0,0 +1,146 @@
+package zio.http.gen.sbt
+
+import java.io.File
+import java.nio.charset.StandardCharsets
+import java.nio.file.StandardOpenOption.{CREATE, TRUNCATE_EXISTING}
+import java.nio.file.{Files, Path}
+
+import scala.io.Source
+
+import zio.json.yaml._
+
+import zio.schema.codec.JsonCodec
+
+import zio.http.endpoint.openapi.OpenAPI
+import zio.http.gen.openapi.{Config, EndpointGen}
+import zio.http.gen.scala.CodeGen
+
+import sbt.Defaults.configSrcSub
+import sbt.Keys._
+import sbt._
+import sbt.util.FileInfo
+
+object ZioHttpCodegen extends AutoPlugin {
+
+  object autoImport {
+
+    val ZIOpenApi = config("oapi") extend Compile
+
+    val zioHttpCodegenMake    = taskKey[Seq[File]]("Generate ADTs & endpoints from OpenAPI spec files")
+    val zioHttpCodegenConf    = settingKey[Config]("Configuration for codegen")
+    val zioHttpCodegenSources = settingKey[File]("Source dir. analoguous to e.g: scalaSource or javaSource")
+  }
+
+  import autoImport._
+
+  override lazy val projectSettings: Seq[Setting[_]] = inConfig(ZIOpenApi)(
+    Seq[Setting[_]](
+      zioHttpCodegenSources := (Compile / sourceDirectory).value / "oapi",
+      sourceGenerators      := Nil,
+      sourceManaged         := configSrcSub(sourceManaged).value,
+      sourceDirectories     := List(zioHttpCodegenSources.value, sourceManaged.value),
+      sources               := {
+        val generatedFiles = Defaults.generate(sourceGenerators).value
+        streams.value.log.info(s"Generated ${generatedFiles.length} OpenAPI spec files")
+        sourceDirectories.value.flatMap(listFilesRec)
+      },
+      zioHttpCodegenConf    := Config.default,
+      zioHttpCodegenMake    := Def.taskDyn {
+
+        val maybeCached = (ZIOpenApi / zioHttpCodegenMake).previous
+        val s           = streams.value
+
+        val cachedCodegen = Tracked.inputChanged[
+          FilesInfo[FileInfo.full.F],
+          Def.Initialize[Task[Seq[File]]],
+        ](s.cacheStoreFactory.make("zioapigen")) { (changed, in) =>
+          maybeCached match {
+            case Some(cached) if !changed =>
+              Def.task {
+                s.log.info("OpenAPI spec unchanged, skipping codegen and using cached files")
+                cached
+              }
+            case _                        =>
+              Def.task {
+                s.log.info("OpenAPI spec changed, or nothing in cache: regenerating code")
+                zioHttpCodegenMakeTask.value
+              }
+          }
+        }
+
+        cachedCodegen(FileInfo.full((ZIOpenApi / sources).value.toSet))
+      }.value,
+    ),
+  ) ++ Seq(
+    Compile / sourceGenerators += (ZIOpenApi / zioHttpCodegenMake).taskValue,
+    Compile / watchSources ++= (ZIOpenApi / sources).value,
+  )
+
+  lazy val zioHttpCodegenMakeTask = Def.task {
+    val openApiFiles: Seq[File]    = (ZIOpenApi / sources).value
+    val openApiRootDirs: Seq[File] = (ZIOpenApi / sourceDirectories).value
+    val baseDir                    = baseDirectory.value
+    val targetDir: File            = (Compile / sourceManaged).value
+    val config: Config             = (ZIOpenApi / zioHttpCodegenConf).value
+
+    openApiFiles.flatMap { openApiFile =>
+      val content        = fileContentAsString(openApiFile)
+      val format         = Format.fromFileName(openApiFile.getName)
+      val openApiRootDir = openApiRootDirs.foldLeft(baseDir) { case (bestSoFar, currentDir) =>
+        val currentPath    = currentDir.getAbsolutePath
+        val isAncestor     = openApiFile.getAbsolutePath.startsWith(currentPath)
+        val isMoreSpecific = currentPath.length >= bestSoFar.getAbsolutePath.length
+        if (isAncestor && isMoreSpecific) currentDir
+        else bestSoFar
+      }
+      val parsedOrError  = format match {
+        case Format.YAML => content.fromYaml[OpenAPI](JsonCodec.jsonDecoder(OpenAPI.schema))
+        case Format.JSON => OpenAPI.fromJson(content)
+      }
+
+      parsedOrError match {
+        case Left(error)    => throw new Exception(s"Failed to parse OpenAPI from $format: $error")
+        case Right(openapi) =>
+          val codegenEndpoints = EndpointGen.fromOpenAPI(openapi, config)
+          val basePackageParts = dirDiffToPackage(openApiRootDir, openApiFile)
+          val currentTargetDir = basePackageParts.foldLeft(targetDir)(_ / _)
+          val currentTargetPat = Path.of(currentTargetDir.toURI())
+
+          CodeGen
+            .renderedFiles(codegenEndpoints, basePackageParts.mkString("."))
+            .map { case (path, content) =>
+              val filePath = currentTargetPat.resolve(path)
+              Files.createDirectories(filePath.getParent)
+              Files.write(filePath, content.getBytes(StandardCharsets.UTF_8), CREATE, TRUNCATE_EXISTING)
+              filePath.toFile
+            }
+            .toSeq
+
+      }
+    }
+  }
+
+  private def listFilesRec(dir: File): List[File] = {
+    def inner(dir: File, acc: List[File]): List[File] =
+      sbt.io.IO.listFiles(dir).foldRight(acc) { case (f, tail) =>
+        if (f.isDirectory) inner(f, tail)
+        else f :: tail
+      }
+
+    inner(dir, Nil)
+  }
+
+  private def fileContentAsString(file: File): String = {
+    val s = Source.fromFile(file)
+    val r = s.mkString
+    s.close()
+    r
+  }
+
+  private def dirDiffToPackage(dir: File, file: File): List[String] = {
+    val dirPath      = dir.toPath
+    val filePath     = file.toPath
+    val relativePath = dirPath.relativize(filePath.getParent)
+    relativePath.toString.split(File.separatorChar).toList
+  }
+}

zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/build.sbt

@@ -0,0 +1,44 @@
+import gigahorse.support.apachehttp.Gigahorse
+import zio.json.ast.Json
+import zio.json.ast.JsonCursor
+import scala.concurrent.{Await, ExecutionContext}
+import scala.concurrent.duration.DurationInt
+import scala.util.{Failure, Try}
+
+lazy val root = (project in file("."))
+  .enablePlugins(ZioHttpCodegen)
+  .settings(
+    name := "uspto-sdk",
+    organization := "com.example",
+    scalaVersion := "2.13.15",
+    libraryDependencies +="dev.zio" %% "zio-http" % "3.0.1",
+    ZIOpenApi / sourceGenerators += Def.task[Seq[File]] {
+
+      val outFile = (ZIOpenApi / sourceManaged).value / "gov" / "uspto" / "ibd" / "api.json"
+      val http = Gigahorse.http(Gigahorse.config)
+      val request = Gigahorse.url("https://developer.uspto.gov/ibd-api/v3/api-docs")
+      val response = http.run(request, Gigahorse.asString)
+
+      Await.result(response.transform(_.flatMap { content =>
+
+        // TODO: this is a temporary workaround
+        //       current zio-http-gen module has many gaps not yet implemented,
+        //       so we need to clean the API just so we can use it here.
+        //       in the future, once the module had matured enough,
+        //       we should remove this part, and perhaps take a more comprehensive example like:
+        //       https://petstore3.swagger.io
+        val either = for {
+          decodedJsObj <- Json.Obj.decoder.decodeJson(content)
+          noInlined404 <- decodedJsObj.delete(JsonCursor.field("paths").isObject.field("/v1/weeklyarchivedata/searchWeeklyArchiveData").isObject.field("get").isObject.field("responses").isObject.field("404"))
+          noInlinedAPI <- noInlined404.delete(JsonCursor.field("paths").isObject.field("/v1/weeklyarchivedata/apistatus"))
+        } yield noInlinedAPI
+
+        either.fold[Try[Seq[File]]](
+          failure => Failure(new Exception(failure)),
+          cleaned => Try {
+            IO.write(outFile, cleaned.toJsonPretty)
+            Seq(outFile)
+          })
+      })(ExecutionContext.global), 1.minute)
+    }
+  )

zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/project/plugins.sbt

@@ -0,0 +1,5 @@
+sys.props.get("plugin.version") match {
+  case Some(ver) => addSbtPlugin("dev.zio" % "zio-http-sbt-codegen" % ver)
+  case None      => sys.error("""|The system property 'plugin.version' is not defined.
+                                 |Specify this property using the scriptedLaunchOpts -D.""".stripMargin)
+}

zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/test

@@ -0,0 +1,7 @@
+# compile should depend on generated sources,
+# thus we'll check it triggers codegen by inspecting expected .class files
+# compiled from expected generated sources
+> compile
+$ exists target/scala-2.13/src_managed/oapi/gov/uspto/ibd/api.json
+$ exists target/scala-2.13/classes/gov/uspto/ibd/v1/weeklyarchivedata/SearchWeeklyArchiveData.class
+$ exists target/scala-2.13/classes/gov/uspto/ibd/component/ArchiveDataRecord.class

zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/build.sbt

@@ -0,0 +1,8 @@
+lazy val root = (project in file("."))
+  .enablePlugins(ZioHttpCodegen)
+  .settings(
+    name := "zoo-sdk",
+    organization := "dev.zoo",
+    scalaVersion := "2.13.15",
+    libraryDependencies +="dev.zio" %% "zio-http" % "3.0.1"
+  )