From e6fa7254fa4085938b0b0a0f5325e577a8ff72e5 Mon Sep 17 00:00:00 2001 From: Russ White <356303+russwyte@users.noreply.github.com> Date: Mon, 23 Dec 2024 00:39:03 -0600 Subject: [PATCH 1/8] Bump netty for CVE-2024-47535 (#3241) Co-authored-by: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> --- project/Dependencies.scala | 2 +- .../src/main/scala/zio/http/netty/server/ServerSSLDecoder.scala | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/project/Dependencies.scala b/project/Dependencies.scala index 9bd09b8f7..267b6f5c2 100644 --- a/project/Dependencies.scala +++ b/project/Dependencies.scala @@ -2,7 +2,7 @@ import sbt.* object Dependencies { val JwtCoreVersion = "10.0.1" - val NettyVersion = "4.1.112.Final" + val NettyVersion = "4.1.116.Final" val NettyIncubatorVersion = "0.0.25.Final" val ScalaCompactCollectionVersion = "2.12.0" val ZioVersion = "2.1.11" diff --git a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerSSLDecoder.scala b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerSSLDecoder.scala index 53c3dfb3f..c3f9a6967 100644 --- a/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerSSLDecoder.scala +++ b/zio-http/jvm/src/main/scala/zio/http/netty/server/ServerSSLDecoder.scala @@ -131,7 +131,7 @@ private[zio] class ServerSSLDecoder(sslConfig: SSLConfig, cfg: Server.Config) ex val httpBehaviour = sslConfig.behaviour if (in.readableBytes < 5) () - else if (SslHandler.isEncrypted(in)) { + else if (SslHandler.isEncrypted(in, false)) { pipeline.replace(this, Names.SSLHandler, sslContext.newHandler(context.alloc())) () } else { From 47bb96a4aa1531abf1a93a7bf9e7d1bb292674c5 Mon Sep 17 00:00:00 2001 From: Jules Ivanic Date: Tue, 24 Dec 2024 17:49:08 +1100 Subject: [PATCH 2/8] Optimise `optBinaryCodec#streamDecoder` (#3237) Co-authored-by: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> --- .../shared/src/main/scala/zio/http/codec/HttpContentCodec.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala index 916742b9e..e846225e5 100644 --- a/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala +++ b/zio-http/shared/src/main/scala/zio/http/codec/HttpContentCodec.scala @@ -172,7 +172,7 @@ sealed trait HttpContentCodec[A] { self => else bc.decode(bytes).map(Some(_)) override def streamDecoder: ZPipeline[Any, DecodeError, Byte, Option[A]] = - ZPipeline.chunks[Byte].map(bc.decode).map(_.toOption) + ZPipeline.chunks[Byte].map(bc.decode(_).toOption) override def streamEncoder: ZPipeline[Any, Nothing, Option[A], Byte] = ZPipeline.identity[Option[A]].map(_.fold(Chunk.empty[Byte])(bc.encode)).flattenChunks From 267c6ce3dbf9de50912d25bc8992b2204b027121 Mon Sep 17 00:00:00 2001 From: Jules Ivanic Date: Wed, 25 Dec 2024 01:56:02 +1100 Subject: [PATCH 3/8] Optimise `zio.http.Response.fromServerSentEvents` (#3238) Co-authored-by: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> --- zio-http/shared/src/main/scala/zio/http/Response.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zio-http/shared/src/main/scala/zio/http/Response.scala b/zio-http/shared/src/main/scala/zio/http/Response.scala index 49825bc06..5793bba2a 100644 --- a/zio-http/shared/src/main/scala/zio/http/Response.scala +++ b/zio-http/shared/src/main/scala/zio/http/Response.scala @@ -213,7 +213,7 @@ object Response { Response( Status.Ok, contentTypeEventStream, - Body.fromCharSequenceStreamChunked(data.map(codec.encode).map(_.asString)), + Body.fromCharSequenceStreamChunked(data.map(codec.encode(_).asString)), ) } From 079f3d8f130c46d67a0b02ac0fbcbd8da5b418aa Mon Sep 17 00:00:00 2001 From: Jules Ivanic Date: Wed, 25 Dec 2024 02:24:01 +1100 Subject: [PATCH 4/8] Optimize some `zio.http.endpoint.Endpoint` code (#3213) Co-authored-by: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> --- .../scala/zio/http/endpoint/Endpoint.scala | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala b/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala index b1b6f6c6f..3eab47c46 100644 --- a/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala +++ b/zio-http/shared/src/main/scala/zio/http/endpoint/Endpoint.scala @@ -67,6 +67,7 @@ final case class Endpoint[PathInput, Input, Err, Output, Auth <: AuthType]( ): HttpCodec[HttpCodecType.RequestType, AuthedInput] = { input ++ authCodec }.asInstanceOf[HttpCodec[HttpCodecType.RequestType, AuthedInput]] + type AuthedInput = authCombiner.Out /** @@ -295,13 +296,11 @@ final case class Endpoint[PathInput, Input, Err, Output, Auth <: AuthType]( self.alternatives.map { case (endpoint, condition) => Handler.fromFunctionZIO { (request: zio.http.Request) => val outputMediaTypes = - NonEmptyChunk - .fromChunk( - request.headers - .getAll(Header.Accept) - .flatMap(_.mimeTypes), - ) - .getOrElse(defaultMediaTypes) + request.headers + .getAll(Header.Accept) + .flatMap(_.mimeTypes) + .nonEmptyOrElse(defaultMediaTypes)(ZIO.identityFn) + (endpoint.input ++ authCodec(endpoint.authType)).decodeRequest(request, config).orDie.flatMap { value => original(value).foldZIO( success = output => Exit.succeed(endpoint.output.encodeResponse(output, outputMediaTypes, config)), @@ -312,15 +311,17 @@ final case class Endpoint[PathInput, Input, Err, Output, Auth <: AuthType]( } // TODO: What to do if there are no endpoints?? - def handlers2(handlers: Chunk[(Handler[Env, Nothing, Request, Response], HttpCodec.Fallback.Condition)]) = - NonEmptyChunk - .fromChunk(handlers) - .getOrElse( - NonEmptyChunk( - Handler.fail(zio.http.Response(status = Status.NotFound)) -> HttpCodec.Fallback.Condition.IsHttpCodecError, - ), + def handlers2( + handlers: Chunk[(Handler[Env, Nothing, Request, Response], HttpCodec.Fallback.Condition)], + ): NonEmptyChunk[(Handler[Env, Response, Request, Response], HttpCodec.Fallback.Condition)] = { + def noFound: NonEmptyChunk[(Handler[Env, Response, Request, Response], HttpCodec.Fallback.Condition)] = + NonEmptyChunk( + Handler.fail(zio.http.Response(status = Status.NotFound)) -> HttpCodec.Fallback.Condition.IsHttpCodecError, ) + handlers.nonEmptyOrElse(ifEmpty = noFound)(ZIO.identityFn) + } + val handler = Handler.fromZIO(CodecConfig.codecRef.get).flatMap { config => val hdlrs = handlers(config) @@ -344,13 +345,12 @@ final case class Endpoint[PathInput, Input, Err, Output, Auth <: AuthType]( val error = cause.defects.head.asInstanceOf[HttpCodecError] val response = { val outputMediaTypes = - NonEmptyChunk - .fromChunk( - request.headers - .getAll(Header.Accept) - .flatMap(_.mimeTypes) :+ MediaTypeWithQFactor(MediaType.application.`json`, Some(0.0)), - ) - .getOrElse(defaultMediaTypes) + ( + request.headers + .getAll(Header.Accept) + .flatMap(_.mimeTypes) :+ MediaTypeWithQFactor(MediaType.application.`json`, Some(0.0)) + ).nonEmptyOrElse(defaultMediaTypes)(ZIO.identityFn) + codecError.encodeResponse(error, outputMediaTypes, config) } ZIO.succeed(response) From 73818915603922d6f518c656ef6e0c21adc553eb Mon Sep 17 00:00:00 2001 From: Gilad Hoch Date: Wed, 25 Dec 2024 12:44:12 +0200 Subject: [PATCH 5/8] [sbt-gen] sbt plugin for openapi to endpoints codegen (#3225) --- .github/workflows/ci.yml | 8 +- build.sbt | 36 +++++ project/BuildHelper.scala | 2 +- project/build.properties | 2 +- .../main/scala/zio/http/gen/sbt/Format.scala | 11 ++ .../zio/http/gen/sbt/ZioHttpCodegen.scala | 146 ++++++++++++++++++ .../zio-http-sbt-codegen/dynamic/build.sbt | 44 ++++++ .../dynamic/project/plugins.sbt | 5 + .../zio-http-sbt-codegen/dynamic/test | 7 + .../zio-http-sbt-codegen/simple/build.sbt | 8 + .../simple/project/plugins.sbt | 5 + .../main/oapi/dev/zoo/service/openapi.yaml | 69 +++++++++ .../sbt-test/zio-http-sbt-codegen/simple/test | 6 + .../test/scala/zio/http/ClientHttpsSpec.scala | 13 +- .../scala/zio/http/endpoint/AuthSpec.scala | 3 +- 15 files changed, 359 insertions(+), 6 deletions(-) create mode 100644 zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/Format.scala create mode 100644 zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/ZioHttpCodegen.scala create mode 100644 zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/build.sbt create mode 100644 zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/project/plugins.sbt create mode 100644 zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/test create mode 100644 zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/build.sbt create mode 100644 zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/project/plugins.sbt create mode 100644 zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/src/main/oapi/dev/zoo/service/openapi.yaml create mode 100644 zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/test diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31c78e038..cfe920e14 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,6 +91,12 @@ jobs: - name: Build project run: sbt '++ ${{ matrix.scala }}' test + - uses: coursier/setup-action@v1 + + - name: Test sbt plugin + if: ${{ github.event_name == 'pull_request' }} && matrix.scala == '2.12.19' + run: sbt ++2.12.19 zioHttpGenSbt/scripted + - uses: coursier/setup-action@v1 with: apps: sbt @@ -106,7 +112,7 @@ jobs: run: sbt '++ ${{ matrix.scala }}' zioHttpShadedTests/test - name: Compress target directories - run: tar cf targets.tar sbt-zio-http-grpc/target zio-http-cli/target target zio-http/jvm/target zio-http-docs/target sbt-zio-http-grpc-tests/target zio-http-gen/target zio-http-benchmarks/target zio-http-tools/target zio-http-example/target zio-http-testkit/target zio-http/js/target zio-http-htmx/target project/target + run: tar cf targets.tar sbt-zio-http-grpc/target zio-http-gen-sbt-plugin/target zio-http-cli/target target zio-http/jvm/target zio-http-docs/target sbt-zio-http-grpc-tests/target zio-http-gen/target zio-http-benchmarks/target zio-http-tools/target zio-http-example/target zio-http-testkit/target zio-http/js/target zio-http-htmx/target project/target - name: Upload target directories uses: actions/upload-artifact@v4 diff --git a/build.sbt b/build.sbt index 71f68af84..79099fff0 100644 --- a/build.sbt +++ b/build.sbt @@ -93,6 +93,22 @@ ThisBuild / githubWorkflowBuildPreamble := Seq( ), ) +ThisBuild / githubWorkflowBuild := { + (ThisBuild / githubWorkflowBuild).value ++ WorkflowJob( + "testSbtPlugin", + "Test sbt plugin", + List( + WorkflowStep.Use(UseRef.Public("coursier", "setup-action", "v1")), + WorkflowStep.Run( + name = Some(s"Test sbt plugin"), + commands = List(s"sbt ++${Scala212} zioHttpGenSbt/scripted"), + cond = Some(s"$${{ github.event_name == 'pull_request' }} && matrix.scala == '$Scala212'"), + ), + ), + scalas = List(Scala212), + ).steps +} + ThisBuild / githubWorkflowBuildPostamble := WorkflowJob( "checkDocGeneration", @@ -350,6 +366,26 @@ lazy val zioHttpGen = (project in file("zio-http-gen")) ) .dependsOn(zioHttpJVM) +lazy val zioHttpGenSbt = (project in file("zio-http-gen-sbt-plugin")) + .enablePlugins(SbtPlugin) + .settings(publishSetting(true)) + .settings( + name := "zio-http-sbt-codegen", + sbtPlugin := true, + scalaVersion := Scala212, + semanticdbEnabled := true, + semanticdbVersion := scalafixSemanticdb.revision, + scalacOptions ++= stdOptions ++ extraOptions(scalaVersion.value), + sbtTestDirectory := sourceDirectory.value / "sbt-test", + scriptedLaunchOpts += ("-Dplugin.version=" + version.value), + scriptedBufferLog := false, + libraryDependencies ++= Seq( + `zio-json-yaml`, + `zio-test`, + `zio-test-sbt`, + ) + ).dependsOn(LocalProject("zioHttpGen")) + lazy val sbtZioHttpGrpc = (project in file("sbt-zio-http-grpc")) .settings(stdSettings("sbt-zio-http-grpc")) .settings(publishSetting(true)) diff --git a/project/BuildHelper.scala b/project/BuildHelper.scala index 44e6243a8..62b15549b 100644 --- a/project/BuildHelper.scala +++ b/project/BuildHelper.scala @@ -12,7 +12,7 @@ object BuildHelper extends ScalaSettings { val ScoverageVersion = "2.0.12" val JmhVersion = "0.4.7" - private val stdOptions = Seq( + val stdOptions = Seq( "-deprecation", "-encoding", "UTF-8", diff --git a/project/build.properties b/project/build.properties index 081fdbbc7..e88a0d817 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.10.0 +sbt.version=1.10.6 diff --git a/zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/Format.scala b/zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/Format.scala new file mode 100644 index 000000000..66268483a --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/Format.scala @@ -0,0 +1,11 @@ +package zio.http.gen.sbt + +sealed trait Format +object Format { + case object YAML extends Format + case object JSON extends Format + def fromFileName(fileName: String): Format = + if (fileName.endsWith("yml") || fileName.endsWith("yaml")) YAML + else if (fileName.endsWith("json")) JSON + else throw new Exception(s"Unknown format for file $fileName") +} diff --git a/zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/ZioHttpCodegen.scala b/zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/ZioHttpCodegen.scala new file mode 100644 index 000000000..d2de9ec11 --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/main/scala/zio/http/gen/sbt/ZioHttpCodegen.scala @@ -0,0 +1,146 @@ +package zio.http.gen.sbt + +import java.io.File +import java.nio.charset.StandardCharsets +import java.nio.file.StandardOpenOption.{CREATE, TRUNCATE_EXISTING} +import java.nio.file.{Files, Path} + +import scala.io.Source + +import zio.json.yaml._ + +import zio.schema.codec.JsonCodec + +import zio.http.endpoint.openapi.OpenAPI +import zio.http.gen.openapi.{Config, EndpointGen} +import zio.http.gen.scala.CodeGen + +import sbt.Defaults.configSrcSub +import sbt.Keys._ +import sbt._ +import sbt.util.FileInfo + +object ZioHttpCodegen extends AutoPlugin { + + object autoImport { + + val ZIOpenApi = config("oapi") extend Compile + + val zioHttpCodegenMake = taskKey[Seq[File]]("Generate ADTs & endpoints from OpenAPI spec files") + val zioHttpCodegenConf = settingKey[Config]("Configuration for codegen") + val zioHttpCodegenSources = settingKey[File]("Source dir. analoguous to e.g: scalaSource or javaSource") + } + + import autoImport._ + + override lazy val projectSettings: Seq[Setting[_]] = inConfig(ZIOpenApi)( + Seq[Setting[_]]( + zioHttpCodegenSources := (Compile / sourceDirectory).value / "oapi", + sourceGenerators := Nil, + sourceManaged := configSrcSub(sourceManaged).value, + sourceDirectories := List(zioHttpCodegenSources.value, sourceManaged.value), + sources := { + val generatedFiles = Defaults.generate(sourceGenerators).value + streams.value.log.info(s"Generated ${generatedFiles.length} OpenAPI spec files") + sourceDirectories.value.flatMap(listFilesRec) + }, + zioHttpCodegenConf := Config.default, + zioHttpCodegenMake := Def.taskDyn { + + val maybeCached = (ZIOpenApi / zioHttpCodegenMake).previous + val s = streams.value + + val cachedCodegen = Tracked.inputChanged[ + FilesInfo[FileInfo.full.F], + Def.Initialize[Task[Seq[File]]], + ](s.cacheStoreFactory.make("zioapigen")) { (changed, in) => + maybeCached match { + case Some(cached) if !changed => + Def.task { + s.log.info("OpenAPI spec unchanged, skipping codegen and using cached files") + cached + } + case _ => + Def.task { + s.log.info("OpenAPI spec changed, or nothing in cache: regenerating code") + zioHttpCodegenMakeTask.value + } + } + } + + cachedCodegen(FileInfo.full((ZIOpenApi / sources).value.toSet)) + }.value, + ), + ) ++ Seq( + Compile / sourceGenerators += (ZIOpenApi / zioHttpCodegenMake).taskValue, + Compile / watchSources ++= (ZIOpenApi / sources).value, + ) + + lazy val zioHttpCodegenMakeTask = Def.task { + val openApiFiles: Seq[File] = (ZIOpenApi / sources).value + val openApiRootDirs: Seq[File] = (ZIOpenApi / sourceDirectories).value + val baseDir = baseDirectory.value + val targetDir: File = (Compile / sourceManaged).value + val config: Config = (ZIOpenApi / zioHttpCodegenConf).value + + openApiFiles.flatMap { openApiFile => + val content = fileContentAsString(openApiFile) + val format = Format.fromFileName(openApiFile.getName) + val openApiRootDir = openApiRootDirs.foldLeft(baseDir) { case (bestSoFar, currentDir) => + val currentPath = currentDir.getAbsolutePath + val isAncestor = openApiFile.getAbsolutePath.startsWith(currentPath) + val isMoreSpecific = currentPath.length >= bestSoFar.getAbsolutePath.length + if (isAncestor && isMoreSpecific) currentDir + else bestSoFar + } + val parsedOrError = format match { + case Format.YAML => content.fromYaml[OpenAPI](JsonCodec.jsonDecoder(OpenAPI.schema)) + case Format.JSON => OpenAPI.fromJson(content) + } + + parsedOrError match { + case Left(error) => throw new Exception(s"Failed to parse OpenAPI from $format: $error") + case Right(openapi) => + val codegenEndpoints = EndpointGen.fromOpenAPI(openapi, config) + val basePackageParts = dirDiffToPackage(openApiRootDir, openApiFile) + val currentTargetDir = basePackageParts.foldLeft(targetDir)(_ / _) + val currentTargetPat = Path.of(currentTargetDir.toURI()) + + CodeGen + .renderedFiles(codegenEndpoints, basePackageParts.mkString(".")) + .map { case (path, content) => + val filePath = currentTargetPat.resolve(path) + Files.createDirectories(filePath.getParent) + Files.write(filePath, content.getBytes(StandardCharsets.UTF_8), CREATE, TRUNCATE_EXISTING) + filePath.toFile + } + .toSeq + + } + } + } + + private def listFilesRec(dir: File): List[File] = { + def inner(dir: File, acc: List[File]): List[File] = + sbt.io.IO.listFiles(dir).foldRight(acc) { case (f, tail) => + if (f.isDirectory) inner(f, tail) + else f :: tail + } + + inner(dir, Nil) + } + + private def fileContentAsString(file: File): String = { + val s = Source.fromFile(file) + val r = s.mkString + s.close() + r + } + + private def dirDiffToPackage(dir: File, file: File): List[String] = { + val dirPath = dir.toPath + val filePath = file.toPath + val relativePath = dirPath.relativize(filePath.getParent) + relativePath.toString.split(File.separatorChar).toList + } +} diff --git a/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/build.sbt b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/build.sbt new file mode 100644 index 000000000..dcab84689 --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/build.sbt @@ -0,0 +1,44 @@ +import gigahorse.support.apachehttp.Gigahorse +import zio.json.ast.Json +import zio.json.ast.JsonCursor +import scala.concurrent.{Await, ExecutionContext} +import scala.concurrent.duration.DurationInt +import scala.util.{Failure, Try} + +lazy val root = (project in file(".")) + .enablePlugins(ZioHttpCodegen) + .settings( + name := "uspto-sdk", + organization := "com.example", + scalaVersion := "2.13.15", + libraryDependencies +="dev.zio" %% "zio-http" % "3.0.1", + ZIOpenApi / sourceGenerators += Def.task[Seq[File]] { + + val outFile = (ZIOpenApi / sourceManaged).value / "gov" / "uspto" / "ibd" / "api.json" + val http = Gigahorse.http(Gigahorse.config) + val request = Gigahorse.url("https://developer.uspto.gov/ibd-api/v3/api-docs") + val response = http.run(request, Gigahorse.asString) + + Await.result(response.transform(_.flatMap { content => + + // TODO: this is a temporary workaround + // current zio-http-gen module has many gaps not yet implemented, + // so we need to clean the API just so we can use it here. + // in the future, once the module had matured enough, + // we should remove this part, and perhaps take a more comprehensive example like: + // https://petstore3.swagger.io + val either = for { + decodedJsObj <- Json.Obj.decoder.decodeJson(content) + noInlined404 <- decodedJsObj.delete(JsonCursor.field("paths").isObject.field("/v1/weeklyarchivedata/searchWeeklyArchiveData").isObject.field("get").isObject.field("responses").isObject.field("404")) + noInlinedAPI <- noInlined404.delete(JsonCursor.field("paths").isObject.field("/v1/weeklyarchivedata/apistatus")) + } yield noInlinedAPI + + either.fold[Try[Seq[File]]]( + failure => Failure(new Exception(failure)), + cleaned => Try { + IO.write(outFile, cleaned.toJsonPretty) + Seq(outFile) + }) + })(ExecutionContext.global), 1.minute) + } + ) \ No newline at end of file diff --git a/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/project/plugins.sbt b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/project/plugins.sbt new file mode 100644 index 000000000..feecfbcc7 --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/project/plugins.sbt @@ -0,0 +1,5 @@ +sys.props.get("plugin.version") match { + case Some(ver) => addSbtPlugin("dev.zio" % "zio-http-sbt-codegen" % ver) + case None => sys.error("""|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) +} \ No newline at end of file diff --git a/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/test b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/test new file mode 100644 index 000000000..f36a96ad0 --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/dynamic/test @@ -0,0 +1,7 @@ +# compile should depend on generated sources, +# thus we'll check it triggers codegen by inspecting expected .class files +# compiled from expected generated sources +> compile +$ exists target/scala-2.13/src_managed/oapi/gov/uspto/ibd/api.json +$ exists target/scala-2.13/classes/gov/uspto/ibd/v1/weeklyarchivedata/SearchWeeklyArchiveData.class +$ exists target/scala-2.13/classes/gov/uspto/ibd/component/ArchiveDataRecord.class diff --git a/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/build.sbt b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/build.sbt new file mode 100644 index 000000000..1233d113f --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/build.sbt @@ -0,0 +1,8 @@ +lazy val root = (project in file(".")) + .enablePlugins(ZioHttpCodegen) + .settings( + name := "zoo-sdk", + organization := "dev.zoo", + scalaVersion := "2.13.15", + libraryDependencies +="dev.zio" %% "zio-http" % "3.0.1" + ) \ No newline at end of file diff --git a/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/project/plugins.sbt b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/project/plugins.sbt new file mode 100644 index 000000000..feecfbcc7 --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/project/plugins.sbt @@ -0,0 +1,5 @@ +sys.props.get("plugin.version") match { + case Some(ver) => addSbtPlugin("dev.zio" % "zio-http-sbt-codegen" % ver) + case None => sys.error("""|The system property 'plugin.version' is not defined. + |Specify this property using the scriptedLaunchOpts -D.""".stripMargin) +} \ No newline at end of file diff --git a/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/src/main/oapi/dev/zoo/service/openapi.yaml b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/src/main/oapi/dev/zoo/service/openapi.yaml new file mode 100644 index 000000000..600b44f8d --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/src/main/oapi/dev/zoo/service/openapi.yaml @@ -0,0 +1,69 @@ +info: + title: Animals Service + version: 0.0.1 +tags: + - name: Animals_API +paths: + /api/v1/zoo/{animal}: + get: + operationId: get_animal + parameters: + - in: path + name: animal + schema: + type: string + required: true + tags: + - Animals_API + description: Get animals by species name + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Animal' +openapi: 3.0.3 +components: + schemas: + Animal: + oneOf: + - $ref: '#/components/schemas/Alligator' + - $ref: '#/components/schemas/Zebra' + AnimalSharedFields: + type: object + required: + - age + - weight + properties: + age: + type: integer + format: int32 + minimum: 0 + weight: + type: number + format: double + minimum: 0 + Alligator: + allOf: + - $ref: '#/components/schemas/AnimalSharedFields' + - type: object + required: + - num_teeth + properties: + num_teeth: + type: integer + format: int32 + minimum: 0 + Zebra: + allOf: + - $ref: '#/components/schemas/AnimalSharedFields' + - type: object + required: + - num_stripes + properties: + num_stripes: + type: integer + format: int32 + minimum: 0 diff --git a/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/test b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/test new file mode 100644 index 000000000..83c430a03 --- /dev/null +++ b/zio-http-gen-sbt-plugin/src/sbt-test/zio-http-sbt-codegen/simple/test @@ -0,0 +1,6 @@ +# compile should depend on generated sources, +# thus we'll check it triggers codegen by inspecting expected .class files +# compiled from expected generated sources +> compile +$ exists target/scala-2.13/classes/dev/zoo/service/api/v1/zoo/Animal.class +$ exists target/scala-2.13/classes/dev/zoo/service/component/Animal.class \ No newline at end of file diff --git a/zio-http/jvm/src/test/scala/zio/http/ClientHttpsSpec.scala b/zio-http/jvm/src/test/scala/zio/http/ClientHttpsSpec.scala index fcdb1842e..3e466bb4d 100644 --- a/zio-http/jvm/src/test/scala/zio/http/ClientHttpsSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/ClientHttpsSpec.scala @@ -18,7 +18,7 @@ package zio.http import zio._ import zio.test.Assertion._ -import zio.test.TestAspect.nonFlaky +import zio.test.TestAspect.{flaky, ignore, nonFlaky} import zio.test.{TestAspect, assertZIO} import zio.http.netty.NettyConfig @@ -53,7 +53,16 @@ abstract class ClientHttpsSpecBase extends ZIOHttpSpec { test("should respond as Bad Request") { val actual = Client.batched(Request.get(badRequest)).map(_.status) assertZIO(actual)(equalTo(Status.BadRequest)) - }, + } @@ ignore /* started getting 503 consistently, + flaky does not help, nor exponential retries. + Either we're being throttled, or the service is under high load. + Regardless, we should not depend on an external service like that. + Luckily, httpbin is available via docker. + So once we make sure to: + + $ docker run -p 80:80 kennethreitz/httpbin + + before invoking tests, we can un-ignore this test. */, test("should throw DecoderException for handshake failure") { val actual = Client.batched(Request.get(untrusted)).exit assertZIO(actual)( diff --git a/zio-http/jvm/src/test/scala/zio/http/endpoint/AuthSpec.scala b/zio-http/jvm/src/test/scala/zio/http/endpoint/AuthSpec.scala index 8e89510c8..e912e7f41 100644 --- a/zio-http/jvm/src/test/scala/zio/http/endpoint/AuthSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/endpoint/AuthSpec.scala @@ -1,6 +1,7 @@ package zio.http.endpoint import zio.Config.Secret +import zio.test.TestAspect.flaky import zio.test._ import zio.{Scope, ZIO, durationInt} @@ -151,7 +152,7 @@ object AuthSpec extends ZIOSpecDefault { .catchAllCause(c => ZIO.logInfoCause(c)) <* ZIO.sleep(1.seconds) response <- response } yield assertTrue(response == "admin") - }, + } @@ flaky, test("Auth basic or bearer with context and endpoint client") { val endpoint = Endpoint(Method.GET / "multiAuth") From 2846f13618b670d2a9e8942a15cdc36c98ef250b Mon Sep 17 00:00:00 2001 From: Eddy Oyieko <67474838+mobley-trent@users.noreply.github.com> Date: Wed, 25 Dec 2024 18:19:48 +0300 Subject: [PATCH 6/8] fix: Parse passwords with arbitrary colons (#3201) Co-authored-by: Nabil Abdel-Hafeez <7283535+987Nabil@users.noreply.github.com> --- .../zio/http/headers/AuthorizationSpec.scala | 15 ++++++++++++++- .../shared/src/main/scala/zio/http/Header.scala | 11 ++++++++--- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/zio-http/jvm/src/test/scala/zio/http/headers/AuthorizationSpec.scala b/zio-http/jvm/src/test/scala/zio/http/headers/AuthorizationSpec.scala index 72d0a2292..144e9aed0 100644 --- a/zio-http/jvm/src/test/scala/zio/http/headers/AuthorizationSpec.scala +++ b/zio-http/jvm/src/test/scala/zio/http/headers/AuthorizationSpec.scala @@ -17,15 +17,18 @@ package zio.http.headers import java.net.URI +import java.util.Base64 import zio.Scope import zio.test._ import zio.http.Header.Authorization -import zio.http.Header.Authorization.Digest +import zio.http.Header.Authorization.{Basic, Digest} import zio.http.ZIOHttpSpec object AuthorizationSpec extends ZIOHttpSpec { + private def encodeCredentials(username: String, password: String): String = + Base64.getEncoder.encodeToString(s"$username:$password".getBytes) override def spec: Spec[TestEnvironment with Scope, Any] = suite("Authorization header suite")( @@ -104,5 +107,15 @@ object AuthorizationSpec extends ZIOHttpSpec { val auth = Authorization.parse("Basic not-base64") assertTrue(auth.isLeft) }, + test("should parse valid Basic Authorization header") { + val encodedHeader = encodeCredentials("user", "pass") + val result = Authorization.parse(s"Basic $encodedHeader") + assertTrue(result.isRight) && assertTrue(result.toOption.get == Basic("user", "pass")) + }, + test("should parse header with multiple colons in password") { + val encodedHeader = encodeCredentials("user", "pass:with:colon") + val result = Authorization.parse(s"Basic $encodedHeader") + assertTrue(result.isRight) && assertTrue(result.toOption.get == Basic("user", "pass:with:colon")) + }, ) } diff --git a/zio-http/shared/src/main/scala/zio/http/Header.scala b/zio-http/shared/src/main/scala/zio/http/Header.scala index 48a87bb8a..bccf90fde 100644 --- a/zio-http/shared/src/main/scala/zio/http/Header.scala +++ b/zio-http/shared/src/main/scala/zio/http/Header.scala @@ -1088,9 +1088,14 @@ object Header { private def parseBasic(value: String): Either[String, Authorization] = { try { - val partsOfBasic = new String(Base64.getDecoder.decode(value)).split(":") - if (partsOfBasic.length == 2) { - Right(Basic(partsOfBasic(0), Secret(partsOfBasic(1)))) + val decoded = new String(Base64.getDecoder.decode(value)) + val indexOfColon = decoded.indexOf(":") + + if (indexOfColon > 0) { + // Extract username as everything before the first ":", and password as everything after + val username = decoded.substring(0, indexOfColon) + val password = decoded.substring(indexOfColon + 1) + Right(Basic(username, Secret(password))) } else { Left("Basic Authorization header value is not in the format username:password") } From 58a90eba8f70efb52c7ccace35e120c65860f01f Mon Sep 17 00:00:00 2001 From: Gilad Hoch Date: Wed, 25 Dec 2024 21:12:30 +0200 Subject: [PATCH 7/8] [gen] adding support for "Any" and "AnyObject" as json in codegen (#3205) [gen] adding support for "Any" and "AnyObject" as Json in codegen logic properly handles correctly these cases: extra_attributes: type: object additionalProperties: true extra_attributes: type: object additionalProperties: {} extra_attributes: additionalProperties: true but fails for this case, which might not be an issue (or an issue with zio-schema): extra_attributes: additionalProperties: {} Also, small code re-arranges to avoid: "Exhaustivity analysis reached max recursion depth, not all missing cases are reported." compile error due to too many cases in pattern match. --- .../zio/http/gen/openapi/EndpointGen.scala | 114 ++++------ .../scala/zio/http/gen/scala/CodeGen.scala | 5 +- .../src/test/resources/AnimalWithAny.scala | 14 ++ .../inline_schema_any_and_any_object.yaml | 49 ++++ .../zio/http/gen/scala/CodeGenSpec.scala | 24 ++ .../http/endpoint/openapi/JsonSchema.scala | 214 ++++++++++-------- 6 files changed, 255 insertions(+), 165 deletions(-) create mode 100644 zio-http-gen/src/test/resources/AnimalWithAny.scala create mode 100644 zio-http-gen/src/test/resources/inline_schema_any_and_any_object.yaml diff --git a/zio-http-gen/src/main/scala/zio/http/gen/openapi/EndpointGen.scala b/zio-http-gen/src/main/scala/zio/http/gen/openapi/EndpointGen.scala index d1cb7a85a..d913d9e29 100644 --- a/zio-http-gen/src/main/scala/zio/http/gen/openapi/EndpointGen.scala +++ b/zio-http-gen/src/main/scala/zio/http/gen/openapi/EndpointGen.scala @@ -673,85 +673,44 @@ final case class EndpointGen(config: Config) { } else None @tailrec - private def schemaToPathCodec(schema: JsonSchema, openAPI: OpenAPI, name: String): Code.PathSegmentCode = { + private def schemaToPathCodec(schema: JsonSchema, openAPI: OpenAPI, name: String): Code.PathSegmentCode = schema match { case JsonSchema.AnnotatedSchema(s, _) => schemaToPathCodec(s, openAPI, name) case JsonSchema.RefSchema(ref) => schemaToPathCodec(resolveSchemaRef(openAPI, ref), openAPI, name) - case JsonSchema.Integer(JsonSchema.IntegerFormat.Int32, _, _, _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.Int) - case JsonSchema.Integer(JsonSchema.IntegerFormat.Int64, _, _, _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.Long) - case JsonSchema.Integer(JsonSchema.IntegerFormat.Timestamp, _, _, _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.Long) - case JsonSchema.String(Some(JsonSchema.StringFormat.UUID), _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.UUID) - case JsonSchema.String(Some(JsonSchema.StringFormat.Date), _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.LocalDate) - case JsonSchema.String(Some(JsonSchema.StringFormat.DateTime), _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.Instant) - case JsonSchema.String(Some(JsonSchema.StringFormat.Time), _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.LocalTime) - case JsonSchema.String(Some(JsonSchema.StringFormat.Duration), _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.Duration) - case JsonSchema.String(_, _, _, _) => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.String) - case JsonSchema.Boolean => - Code.PathSegmentCode(name = name, segmentType = Code.CodecType.Boolean) - case JsonSchema.OneOfSchema(_) => throw new Exception("Alternative path variables are not supported") - case JsonSchema.AllOfSchema(_) => throw new Exception("Path variables must have exactly one schema") - case JsonSchema.AnyOfSchema(_) => throw new Exception("Path variables must have exactly one schema") - case JsonSchema.Number(_, _, _, _, _, _) => - throw new Exception("Floating point path variables are currently not supported") - case JsonSchema.ArrayType(_, _, _) => throw new Exception("Array path variables are not supported") - case JsonSchema.Object(_, _, _) => throw new Exception("Object path variables are not supported") - case JsonSchema.Enum(_) => throw new Exception("Enum path variables are not supported") - case JsonSchema.Null => throw new Exception("Null path variables are not supported") - case JsonSchema.AnyJson => throw new Exception("AnyJson path variables are not supported") + case s: JsonSchema.Integer => Code.PathSegmentCode(name = name, segmentType = integerCodec(s.format)) + case s: JsonSchema.String => Code.PathSegmentCode(name = name, segmentType = stringCodec(s.format)) + case JsonSchema.Boolean => Code.PathSegmentCode(name = name, segmentType = Code.CodecType.Boolean) + case JsonSchema.OneOfSchema(_) => throw new Exception("Alternative path variables are not supported") + case JsonSchema.AllOfSchema(_) => throw new Exception("Path variables must have exactly one schema") + case JsonSchema.AnyOfSchema(_) => throw new Exception("Path variables must have exactly one schema") + case JsonSchema.ArrayType(_, _, _) => throw new Exception("Array path variables are not supported") + case JsonSchema.Object(_, _, _) => throw new Exception("Object path variables are not supported") + case JsonSchema.Enum(_) => throw new Exception("Enum path variables are not supported") + case JsonSchema.Null => throw new Exception("Null path variables are not supported") + case JsonSchema.AnyJson => throw new Exception("AnyJson path variables are not supported") + case _: JsonSchema.Number => throw new Exception("Floating point path variables are not supported") } - } @tailrec private def schemaToQueryParamCodec( schema: JsonSchema, openAPI: OpenAPI, name: String, - ): Code.QueryParamCode = { - schema match { - case JsonSchema.AnnotatedSchema(s, _) => - schemaToQueryParamCodec(s, openAPI, name) - case JsonSchema.RefSchema(ref) => - schemaToQueryParamCodec(resolveSchemaRef(openAPI, ref), openAPI, name) - case JsonSchema.Integer(JsonSchema.IntegerFormat.Int32, _, _, _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.Int) - case JsonSchema.Integer(JsonSchema.IntegerFormat.Int64, _, _, _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.Long) - case JsonSchema.Integer(JsonSchema.IntegerFormat.Timestamp, _, _, _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.Long) - case JsonSchema.String(Some(JsonSchema.StringFormat.Date), _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.LocalDate) - case JsonSchema.String(Some(JsonSchema.StringFormat.DateTime), _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.Instant) - case JsonSchema.String(Some(JsonSchema.StringFormat.Duration), _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.Duration) - case JsonSchema.String(Some(JsonSchema.StringFormat.Time), _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.LocalTime) - case JsonSchema.String(Some(JsonSchema.StringFormat.UUID), _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.UUID) - case JsonSchema.String(_, _, _, _) => - Code.QueryParamCode(name = name, queryType = Code.CodecType.String) - case JsonSchema.Boolean => - Code.QueryParamCode(name = name, queryType = Code.CodecType.Boolean) - case JsonSchema.OneOfSchema(_) => throw new Exception("Alternative query parameters are not supported") - case JsonSchema.AllOfSchema(_) => throw new Exception("Query parameters must have exactly one schema") - case JsonSchema.AnyOfSchema(_) => throw new Exception("Query parameters must have exactly one schema") - case JsonSchema.Number(_, _, _, _, _, _) => - throw new Exception("Floating point query parameters are currently not supported") - case JsonSchema.ArrayType(_, _, _) => throw new Exception("Array query parameters are not supported") - case JsonSchema.Object(_, _, _) => throw new Exception("Object query parameters are not supported") - case JsonSchema.Enum(_) => throw new Exception("Enum query parameters are not supported") - case JsonSchema.Null => throw new Exception("Null query parameters are not supported") - case JsonSchema.AnyJson => throw new Exception("AnyJson query parameters are not supported") - } + ): Code.QueryParamCode = schema match { + case JsonSchema.AnnotatedSchema(s, _) => schemaToQueryParamCodec(s, openAPI, name) + case JsonSchema.RefSchema(ref) => schemaToQueryParamCodec(resolveSchemaRef(openAPI, ref), openAPI, name) + case JsonSchema.Boolean => Code.QueryParamCode(name = name, queryType = Code.CodecType.Boolean) + case s: JsonSchema.Integer => Code.QueryParamCode(name = name, queryType = integerCodec(s.format)) + case s: JsonSchema.String => Code.QueryParamCode(name = name, queryType = stringCodec(s.format)) + case _: JsonSchema.Number => throw new Exception("Floating point query parameters are not supported") + case JsonSchema.OneOfSchema(_) => throw new Exception("Alternative query parameters are not supported") + case JsonSchema.AllOfSchema(_) => throw new Exception("Query parameters must have exactly one schema") + case JsonSchema.AnyOfSchema(_) => throw new Exception("Query parameters must have exactly one schema") + case JsonSchema.ArrayType(_, _, _) => throw new Exception("Array query parameters are not supported") + case JsonSchema.Object(_, _, _) => throw new Exception("Object query parameters are not supported") + case JsonSchema.Enum(_) => throw new Exception("Enum query parameters are not supported") + case JsonSchema.Null => throw new Exception("Null query parameters are not supported") + case JsonSchema.AnyJson => throw new Exception("AnyJson query parameters are not supported") } private def fieldsOfObject(openAPI: OpenAPI, annotations: Chunk[JsonSchema.MetaData])( @@ -1042,6 +1001,7 @@ final case class EndpointGen(config: Config) { properties.map { case (name, schema) => name -> schema.withoutAnnotations }.collect { case (name, schema) if !schema.isInstanceOf[JsonSchema.RefSchema] + && !(schema == JsonSchema.AnyJson) && !schema.isPrimitive && !schema.isCollection => schemaToCode(schema, openAPI, name.capitalize, Chunk.empty) @@ -1093,7 +1053,7 @@ final case class EndpointGen(config: Config) { ), ) case JsonSchema.Null => throw new Exception("Null query parameters are not supported") - case JsonSchema.AnyJson => throw new Exception("AnyJson query parameters are not supported") + case JsonSchema.AnyJson => None } } @@ -1378,4 +1338,18 @@ final case class EndpointGen(config: Config) { } } + private def integerCodec(format: JsonSchema.IntegerFormat): Code.CodecType = format match { + case JsonSchema.IntegerFormat.Int32 => Code.CodecType.Int + case JsonSchema.IntegerFormat.Int64 => Code.CodecType.Long + case JsonSchema.IntegerFormat.Timestamp => Code.CodecType.Long + } + + private def stringCodec(format: Option[JsonSchema.StringFormat]): Code.CodecType = format match { + case Some(JsonSchema.StringFormat.Date) => Code.CodecType.LocalDate + case Some(JsonSchema.StringFormat.DateTime) => Code.CodecType.Instant + case Some(JsonSchema.StringFormat.Duration) => Code.CodecType.Duration + case Some(JsonSchema.StringFormat.Time) => Code.CodecType.LocalTime + case Some(JsonSchema.StringFormat.UUID) => Code.CodecType.UUID + case _ => Code.CodecType.String + } } diff --git a/zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala b/zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala index 6b582ca7d..bda36f4f7 100644 --- a/zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala +++ b/zio-http-gen/src/main/scala/zio/http/gen/scala/CodeGen.scala @@ -4,8 +4,6 @@ import java.nio.charset.StandardCharsets import java.nio.file.StandardOpenOption._ import java.nio.file._ -import scala.util.matching.Regex - object CodeGen { private val EndpointImports = @@ -265,6 +263,9 @@ object CodeGen { case Code.TypeRef(name) => Nil -> name + case Code.ScalaType.JsonAST => + List(Code.Import("zio.json.ast.Json")) -> "Json" + case scalaType => throw new Exception(s"Unknown ScalaType: $scalaType") } diff --git a/zio-http-gen/src/test/resources/AnimalWithAny.scala b/zio-http-gen/src/test/resources/AnimalWithAny.scala new file mode 100644 index 000000000..119f0c684 --- /dev/null +++ b/zio-http-gen/src/test/resources/AnimalWithAny.scala @@ -0,0 +1,14 @@ +package test.component + +import zio.json.ast.Json +import zio.schema._ +import zio.schema.annotation.fieldName + +case class Animal( + name: String, + eats: Json, + @fieldName("extra_attributes") extraAttributes: Map[String, Json], +) +object Animal { + implicit val codec: Schema[Animal] = DeriveSchema.gen[Animal] +} diff --git a/zio-http-gen/src/test/resources/inline_schema_any_and_any_object.yaml b/zio-http-gen/src/test/resources/inline_schema_any_and_any_object.yaml new file mode 100644 index 000000000..fbe3b9c2f --- /dev/null +++ b/zio-http-gen/src/test/resources/inline_schema_any_and_any_object.yaml @@ -0,0 +1,49 @@ +info: + title: Animals Service + version: 0.0.1 +tags: + - name: Animals_API +paths: + /api/v1/zoo/{animal}: + get: + operationId: get_animal + parameters: + - in: path + name: animal + schema: + type: string + required: true + tags: + - Animals_API + description: Get animals by species name + responses: + "200": + content: + application/json: + schema: + type: array + items: + $ref: '#/components/schemas/Animal' + description: OK + "500": + content: + application/json: + schema: + $ref: '#/components/schemas/HttpError' + description: Internal Server Error +openapi: 3.0.3 +components: + schemas: + Animal: + type: object + required: + - name + - eats + - extra_attributes + properties: + name: + type: string + eats: {} + extra_attributes: + type: object + additionalProperties: true \ No newline at end of file diff --git a/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala b/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala index fe0aa3f37..fda70f5ff 100644 --- a/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala +++ b/zio-http-gen/src/test/scala/zio/http/gen/scala/CodeGenSpec.scala @@ -1012,6 +1012,30 @@ object CodeGenSpec extends ZIOSpecDefault { } } } @@ TestAspect.exceptScala3, + test("Schema with any and any object") { + val openAPIString = stringFromResource("/inline_schema_any_and_any_object.yaml") + + openApiFromYamlString(openAPIString) { oapi => + codeGenFromOpenAPI( + oapi, + Config.default.copy( + fieldNamesNormalization = Config.default.fieldNamesNormalization.copy(enableAutomatic = true), + ), + ) { testDir => + allFilesShouldBe( + testDir.toFile, + List( + "api/v1/zoo/Animal.scala", + "component/Animal.scala", + ), + ) && fileShouldBe( + testDir, + "component/Animal.scala", + "/AnimalWithAny.scala", + ) + } + } + } @@ TestAspect.exceptScala3, test("Generate all responses") { val oapi = OpenAPI( diff --git a/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/JsonSchema.scala b/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/JsonSchema.scala index 3ccac01aa..8dced7348 100644 --- a/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/JsonSchema.scala +++ b/zio-http/shared/src/main/scala/zio/http/endpoint/openapi/JsonSchema.scala @@ -254,111 +254,139 @@ object JsonSchema { .get private[openapi] def fromSerializableSchema(schema: SerializableJsonSchema): JsonSchema = { - val additionalProperties = schema.additionalProperties match { - case Some(BoolOrSchema.BooleanWrapper(bool)) => Left(bool) - case Some(BoolOrSchema.SchemaWrapper(schema)) => - val valuesSchema = fromSerializableSchema(schema) - Right( - schema.optionalKeySchema.fold(valuesSchema)(keySchema => - valuesSchema.annotate( - MetaData.KeySchema( - fromSerializableSchema(keySchema), + + val definedAttributesCount = schema.productIterator.count(_.asInstanceOf[Option[_]].isDefined) + + // if type: object with additionalProperties defined, + // but nothing else, we should assume a free form object + // if type is not defined, but additionalProperties is, + // and nothing else, object is assumed again. + // if both type: object and additionalProperties are defined, + // and nothing else, object is assumed. + def anyObject: Boolean = { + val isObject = schema.schemaType.contains(TypeOrTypes.Type("object")) + val hasAttrs = schema.additionalProperties.collect { case BoolOrSchema.BooleanWrapper(b) => + b + }.exists(identity) + + // if definedAttributesCount == 0, this also yields true, + // but we check for it before calling this function, + // thus no need to check it here. + val isAnyObj = List(isObject, hasAttrs).count(identity) == definedAttributesCount + + isAnyObj + } + + if (definedAttributesCount == 0) JsonSchema.AnyJson + else if (anyObject) JsonSchema.Object(Map.empty, Right(JsonSchema.AnyJson), Chunk.empty) + else { + + val additionalProperties = schema.additionalProperties match { + case Some(BoolOrSchema.BooleanWrapper(bool)) => Left(bool) + case Some(BoolOrSchema.SchemaWrapper(schema)) => + val valuesSchema = fromSerializableSchema(schema) + Right( + schema.optionalKeySchema.fold(valuesSchema)(keySchema => + valuesSchema.annotate( + MetaData.KeySchema( + fromSerializableSchema(keySchema), + ), ), ), - ), - ) - case None => Left(true) - } + ) + case None => Left(true) + } - var jsonSchema: JsonSchema = schema match { - case schema if schema.ref.isDefined => - RefSchema(schema.ref.get) - case schema if schema.schemaType.contains(TypeOrTypes.Type("number")) => - JsonSchema.Number( - NumberFormat.fromString(schema.format.getOrElse("double")), - schema.minimum.map(_.fold(identity, _.toDouble)), - schema.exclusiveMinimum.map(_.map(_.fold(identity, _.toDouble))), - schema.maximum.map(_.fold(identity, _.toDouble)), - schema.exclusiveMaximum.map(_.map(_.fold(identity, _.toDouble))), - ) - case schema if schema.schemaType.contains(TypeOrTypes.Type("integer")) => - JsonSchema.Integer( - IntegerFormat.fromString(schema.format.getOrElse("int64")), - schema.minimum.map(_.fold(_.toLong, identity)), - schema.exclusiveMinimum.map(_.map(_.fold(_.toLong, identity))), - schema.maximum.map(_.fold(_.toLong, identity)), - schema.exclusiveMaximum.map(_.map(_.fold(_.toLong, identity))), - ) - case schema if schema.schemaType.contains(TypeOrTypes.Type("string")) && schema.enumValues.isEmpty => - JsonSchema.String( - schema.format.map(StringFormat.fromString), - schema.pattern.map(Pattern.apply), - schema.maxLength, - schema.minLength, - ) - case schema if schema.schemaType.contains(TypeOrTypes.Type("boolean")) => - JsonSchema.Boolean - case schema if schema.schemaType.contains(TypeOrTypes.Type("array")) => - JsonSchema.ArrayType( - schema.items.map(fromSerializableSchema), - schema.minItems, - schema.uniqueItems.contains(true), - ) - case schema if schema.enumValues.isDefined => - JsonSchema.Enum(schema.enumValues.get.map(EnumValue.fromJson)) - case schema if schema.oneOf.isDefined => - OneOfSchema(schema.oneOf.get.map(fromSerializableSchema)) - case schema if schema.allOf.isDefined => - AllOfSchema(schema.allOf.get.map(fromSerializableSchema)) - case schema if schema.anyOf.isDefined => - AnyOfSchema(schema.anyOf.get.map(fromSerializableSchema)) - case schema if schema.schemaType.contains(TypeOrTypes.Type("null")) => - JsonSchema.Null - case schema if schema.schemaType.contains(TypeOrTypes.Type("object")) || schema.schemaType.isEmpty => - JsonSchema.Object( - schema.properties - .map(_.map { case (name, schema) => name -> fromSerializableSchema(schema) }) - .getOrElse(Map.empty), - additionalProperties, - schema.required.getOrElse(Chunk.empty), - ) - case _ => - throw new IllegalArgumentException(s"Can't convert $schema") - } + var jsonSchema: JsonSchema = schema match { + case schema if schema.ref.isDefined => + RefSchema(schema.ref.get) + case schema if schema.schemaType.contains(TypeOrTypes.Type("number")) => + JsonSchema.Number( + NumberFormat.fromString(schema.format.getOrElse("double")), + schema.minimum.map(_.fold(identity, _.toDouble)), + schema.exclusiveMinimum.map(_.map(_.fold(identity, _.toDouble))), + schema.maximum.map(_.fold(identity, _.toDouble)), + schema.exclusiveMaximum.map(_.map(_.fold(identity, _.toDouble))), + ) + case schema if schema.schemaType.contains(TypeOrTypes.Type("integer")) => + JsonSchema.Integer( + IntegerFormat.fromString(schema.format.getOrElse("int64")), + schema.minimum.map(_.fold(_.toLong, identity)), + schema.exclusiveMinimum.map(_.map(_.fold(_.toLong, identity))), + schema.maximum.map(_.fold(_.toLong, identity)), + schema.exclusiveMaximum.map(_.map(_.fold(_.toLong, identity))), + ) + case schema if schema.schemaType.contains(TypeOrTypes.Type("string")) && schema.enumValues.isEmpty => + JsonSchema.String( + schema.format.map(StringFormat.fromString), + schema.pattern.map(Pattern.apply), + schema.maxLength, + schema.minLength, + ) + case schema if schema.schemaType.contains(TypeOrTypes.Type("boolean")) => + JsonSchema.Boolean + case schema if schema.schemaType.contains(TypeOrTypes.Type("array")) => + JsonSchema.ArrayType( + schema.items.map(fromSerializableSchema), + schema.minItems, + schema.uniqueItems.contains(true), + ) + case schema if schema.enumValues.isDefined => + JsonSchema.Enum(schema.enumValues.get.map(EnumValue.fromJson)) + case schema if schema.oneOf.isDefined => + OneOfSchema(schema.oneOf.get.map(fromSerializableSchema)) + case schema if schema.allOf.isDefined => + AllOfSchema(schema.allOf.get.map(fromSerializableSchema)) + case schema if schema.anyOf.isDefined => + AnyOfSchema(schema.anyOf.get.map(fromSerializableSchema)) + case schema if schema.schemaType.contains(TypeOrTypes.Type("null")) => + JsonSchema.Null + case schema if schema.schemaType.contains(TypeOrTypes.Type("object")) || schema.schemaType.isEmpty => + JsonSchema.Object( + schema.properties + .map(_.map { case (name, schema) => name -> fromSerializableSchema(schema) }) + .getOrElse(Map.empty), + additionalProperties, + schema.required.getOrElse(Chunk.empty), + ) + case _ => + throw new IllegalArgumentException(s"Can't convert $schema") + } - val examples = Chunk.fromIterable(schema.example) ++ schema.examples.getOrElse(Chunk.empty) - if (examples.nonEmpty) jsonSchema = jsonSchema.examples(examples) + val examples = Chunk.fromIterable(schema.example) ++ schema.examples.getOrElse(Chunk.empty) + if (examples.nonEmpty) jsonSchema = jsonSchema.examples(examples) - schema.description match { - case Some(value) => jsonSchema = jsonSchema.description(value) - case None => () - } + schema.description match { + case Some(value) => jsonSchema = jsonSchema.description(value) + case None => () + } - schema.nullable match { - case Some(value) => jsonSchema = jsonSchema.nullable(value) - case None => () - } + schema.nullable match { + case Some(value) => jsonSchema = jsonSchema.nullable(value) + case None => () + } - schema.discriminator match { - case Some(value) => jsonSchema = jsonSchema.discriminator(value) - case None => () - } + schema.discriminator match { + case Some(value) => jsonSchema = jsonSchema.discriminator(value) + case None => () + } - schema.contentEncoding.flatMap(ContentEncoding.fromString) match { - case Some(value) => jsonSchema = jsonSchema.contentEncoding(value) - case None => () - } + schema.contentEncoding.flatMap(ContentEncoding.fromString) match { + case Some(value) => jsonSchema = jsonSchema.contentEncoding(value) + case None => () + } - schema.contentMediaType match { - case Some(value) => jsonSchema = jsonSchema.contentMediaType(value) - case None => () - } + schema.contentMediaType match { + case Some(value) => jsonSchema = jsonSchema.contentMediaType(value) + case None => () + } - jsonSchema = jsonSchema.default(schema.default) + jsonSchema = jsonSchema.default(schema.default) - jsonSchema = jsonSchema.deprecated(schema.deprecated.getOrElse(false)) + jsonSchema = jsonSchema.deprecated(schema.deprecated.getOrElse(false)) - jsonSchema + jsonSchema + } } def fromTextCodec(codec: TextCodec[_]): JsonSchema = From 6d9dce1791d9e25ad207f89cfafdf973c08c1eaa Mon Sep 17 00:00:00 2001 From: Gilad Hoch Date: Fri, 27 Dec 2024 08:37:52 +0200 Subject: [PATCH 8/8] added basic documentation for the openapi codegen sbt plugin (#3262) --- docs/reference/openapi-gen-sbt-plugin.md | 75 ++++++++++++++++++++++++ 1 file changed, 75 insertions(+) create mode 100644 docs/reference/openapi-gen-sbt-plugin.md diff --git a/docs/reference/openapi-gen-sbt-plugin.md b/docs/reference/openapi-gen-sbt-plugin.md new file mode 100644 index 000000000..7af37bb7e --- /dev/null +++ b/docs/reference/openapi-gen-sbt-plugin.md @@ -0,0 +1,75 @@ +--- +id: openapi-gen-sbt-plugin +title: OpenAPI codegen sbt plugin +--- + +This plugin allows to easily generate scala source code with zio-http Endpoints from OpenAPI spec files. + +## How to use + +The plugin offers 2 modes of operation that can be mixed and used together: +- Generating from unmanaged static OpenAPI spec files +- Generating from managed dynamic OpenAPI spec files + +in `project/plugins.sbt` add the following line: +```scala +addSbtPlugin("dev.zio" % "zio-http-sbt-codegen" % "@VERSION@") // make sure the version of the sbt plugin + // matches the version of zio-http you are using +``` + +in `build.sbt` enable the plugin by adding: +```scala +enablePlugins(ZioHttpCodegen) +``` + +### 1. Generating from unmanaged static OpenAPI spec files +Place your manually curated OpenAPI spec files (`.yml`, `.yaml`, or `.json`) in `src/main/oapi//`.\ +That's it. No other configuration is needed for basic usage. \ +Once you `compile` your project, the `zioHttpCodegenMake` task is automatically invoked, and the generated code will be placed under `target/scala-/src_managed/main/scala`. + +### 2. Generating from managed dynamic OpenAPI spec files +In this mode, you can hook into `ZIOpenApi / sourceGenerators` a task to generate OpenAPI spec file, exactly like you would do with regular `Compile / sourceGenerators` for scala source files. +You might have some OpenAPI spec files hosted on [swaggerhub](https://app.swaggerhub.com/) or a similar service, +or maybe you use services that expose OpenAPI specs via REST API, or perhaps you have a local project that can build its own spec and you want to run the spec generate command. +Whatever the scenario you're dealing with, it can be very handy to dynamically fetch/generate the latest most updated spec file, so the generated code stays up to date with any changes introduced. + +Here's how you can do it: +```scala +import gigahorse.support.apachehttp.Gigahorse +import scala.concurrent.Await +import scala.concurrent.duration.DurationInt + +ZIOpenApi / sourceGenerators += Def.task[Seq[File]] { + // we'll fetch a spec from https://www.petstore.dev/ + // gigahorse comes builtin with sbt, but any other http client can be used + val http = Gigahorse.http(Gigahorse.config) + val request = Gigahorse.url("https://raw.githubusercontent.com/readmeio/oas-examples/main/3.0/yaml/response-http-behavior.yaml") + val response = http.run(request, Gigahorse.asString) + val content = Await.result(response, 1.minute) + + // path under target/scala-/src_managed/oapi/ + // corresponds to the package where scala sources will be generated + val outFile = (ZIOpenApi / sourceManaged).value / "dev" / "petstore" / "http" / "test" / "api.yaml" + IO.write(outFile, content) + + // as long the task yields a Seq[File] of valid OpenAPI spec files, + // and those files follow the path structure `src_managed/oapi//`, + // the plugin will pick it up, and generate the corresponding scala sources. + Seq(outFile) +} +``` + +## Configuration +The plugin offers a setting key which you can set to control how code is generated: +```scala +zioHttpCodegenConf := zio.http.gen.openapi.Config.default +``` + +## Caveats +The plugin allows you to provide multiple files. +Note that if you place multiple files in the same directory, +which means same package for the generated code - you must make sure there are no "collisions" between generated classes. +If the same class is going to be generated differently in different files, you probably want to have a different package for it. + +Also, please note that the plugin relies on the file extension to determine how to parse it. +So files must have the correct extension (`.yml`, `.yaml`, or `.json`), and the content must be formatted accordingly.