Initial version

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/cd.yml

@@ -0,0 +1,114 @@
+name: CD
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Deployment environment (dev|stage|prod)'
+        required: true
+        default: 'dev'
+
+  push:
+    branches:
+      - develop
+      - release
+
+permissions:
+  contents: read
+
+env:
+  JAVA_VERSION: 21
+  DOCKER_REGISTRY: ghcr.io
+  DOCKER_IMAGE_NAME: ${{ github.repository }}
+  HTRC_NEXUS_DRHTRC_PWD: ${{ secrets.HTRC_NEXUS_DRHTRC_PWD }}
+
+jobs:
+  docker-build:
+    name: Build and push Docker image
+    runs-on: ubuntu-latest
+    permissions:
+      packages: write
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '${{ env.JAVA_VERSION }}'
+          distribution: 'temurin'
+          cache: 'sbt'
+      - name: Generate Dockerfile
+        run: sbt "Docker/stage"
+      - name: Show GitHub context
+        env:
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+        run: echo "$GITHUB_CONTEXT"
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_REGISTRY }}/${{ env.DOCKER_IMAGE_NAME }}
+          tags: |
+            type=sha,prefix={{branch}}-,priority=750,enable=${{ startsWith(github.ref, 'refs/heads/') }}
+            type=ref,event=branch
+            type=ref,event=pr
+            type=pep440,pattern={{version}}
+            type=pep440,pattern={{major}}.{{minor}}
+            type=raw,value=latest,enable={{is_default_branch}}
+          labels: |
+            org.opencontainers.image.vendor=HathiTrust Research Center
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.DOCKER_REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push Docker images
+        uses: docker/build-push-action@v5
+        with:
+          context: target/docker/stage/
+          push: true
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+      - name: Repository Dispatch
+        if: github.event.inputs.environment == ''
+        uses: peter-evans/repository-dispatch@v3
+        env:
+          HEAD_COMMIT_MESSAGE: ${{ toJSON(github.event.head_commit.message) }}
+        with:
+          token: ${{ secrets.PAT }}
+          repository: htrc/torchlite-argocd
+          event-type: argocd
+          client-payload: >-
+            {
+              "image": "${{ fromJSON(steps.meta.outputs.json).tags[0] }}",
+              "commit_msg": ${{ env.HEAD_COMMIT_MESSAGE }},
+              "repository": "${{ github.event.repository.name }}",
+              "environment": "${{ github.event.inputs.environment }}",
+              "ref": "${{ github.head_ref || github.ref_name }}",
+              "actor": "${{ github.triggering_actor }}"
+            }
+      - name: Manual Repository Dispatch
+        if: github.event.inputs.environment != ''
+        uses: peter-evans/repository-dispatch@v3
+        env:
+          HEAD_COMMIT_MESSAGE: ${{ toJSON(github.event.head_commit.message) }}
+        with:
+          token: ${{ secrets.PAT }}
+          repository: htrc/torchlite-argocd
+          event-type: argocd-manual
+          client-payload: >-
+            {
+              "image": "${{ fromJSON(steps.meta.outputs.json).tags[0] }}",
+              "commit_msg": ${{ env.HEAD_COMMIT_MESSAGE }},
+              "repository": "${{ github.event.repository.name }}",
+              "environment": "${{ github.event.inputs.environment }}",
+              "ref": "${{ github.head_ref || github.ref_name }}",
+              "actor": "${{ github.triggering_actor }}"
+            }

.github/workflows/ci.yml

@@ -0,0 +1,33 @@
+name: CI
+
+on: [pull_request]
+
+permissions:
+  contents: read
+
+env:
+  JAVA_VERSION: 21
+  HTRC_NEXUS_DRHTRC_PWD: ${{ secrets.HTRC_NEXUS_DRHTRC_PWD }}
+
+jobs:
+  test:
+    name: Build & test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Java
+        uses: actions/setup-java@v4
+        with:
+          java-version: '${{ env.JAVA_VERSION }}'
+          distribution: 'temurin'
+          cache: 'sbt'
+      - name: Run tests with coverage
+        run: sbt -mem 4000 coverage "+test" coverageReport
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          fail_ci_if_error: false

.github/workflows/dependency-graph.yml

@@ -0,0 +1,16 @@
+name: Update Dependency Graph
+on:
+  push:
+    branches:
+      - main
+
+env:
+  HTRC_NEXUS_DRHTRC_PWD: ${{ secrets.HTRC_NEXUS_DRHTRC_PWD }}
+
+jobs:
+  dependency-graph:
+    name: Update Dependency Graph
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: scalacenter/sbt-dependency-submission@v2

.gitignore

@@ -0,0 +1,13 @@
+/.clover/
+/.idea/
+*.iml
+/.bsp/
+/.metals/
+.bloop/
+metals.sbt
+*.sublime-workspace
+target/
+project/project/
+/RUNNING_PID
+/logs/
+*.log

README.md

@@ -0,0 +1,31 @@
+[![Scala CI](https://github.com/htrc/HTRC-EF-API/actions/workflows/ci.yml/badge.svg)](https://github.com/htrc/HTRC-EF-API/actions/workflows/ci.yml)
+[![codecov](https://codecov.io/gh/htrc/HTRC-EF-API/graph/badge.svg?token=SHgcExcM7o)](https://codecov.io/gh/htrc/HTRC-EF-API)
+[![GitHub release (latest SemVer including pre-releases)](https://img.shields.io/github/v/release/htrc/HTRC-EF-API?include_prereleases&sort=semver)](https://github.com/htrc/HTRC-EF-API/releases/latest)
+
+# HTRC-ExtractedFeatures-API
+An API for accessing the HTRC Extracted Features dataset
+
+# Build
+`sbt clean stage`
+Then find the result in `target/universal/stage/`
+
+# Deploy
+Copy the folder `target/universal/stage/` to the deployment location and rename as desired (henceforth referred to as `DEPLOY_DIR`).
+
+# Setup
+0. Generate an application secret by running `sbt playGenerateSecret`
+1. Set `EFAPI_MONGODB_URI` environment variable to point to the Mongo instance holding the EF data
+2. Set `EFAPI_SECRET` environment variable to the value generated by step 0
+
+(alternatively, these settings can also be configured by editing `target/universal/stage/conf/application.conf`)
+
+# Run
+*Note:* You must have the environment variables set before running (or edited the `application.conf` accordingly)
+```bash
+$DEPLOY_DIR/bin/htrc-extractedfeatures-api -Dhttp.address=HOST -Dhttp.port=PORT -Dplay.http.context=/api
+```
+where `HOST` is the desired hostname or IP to bind to, and `PORT` is the desired port to run on.
+
+# API
+
+https://htrc.stoplight.io/docs/torchlite-ef-api

app/ErrorHandler.scala

@@ -0,0 +1,26 @@
+import exceptions.{ApiException, ErrorCodes}
+import play.api._
+import play.api.http.DefaultHttpErrorHandler
+import play.api.mvc._
+import play.api.routing.Router
+import protocol.WrappedResponse
+
+import javax.inject._
+import scala.concurrent._
+
+@Singleton
+class ErrorHandler @Inject()(env: Environment,
+                             config: Configuration,
+                             sourceMapper: OptionalSourceMapper,
+                             router: Provider[Router]) extends DefaultHttpErrorHandler(env, config, sourceMapper, router) {
+  val logger: Logger = Logger(getClass)
+
+  override def onServerError(request: RequestHeader, exception: Throwable): Future[Result] = {
+    logger.error(request.toString(), exception)
+
+    exception match {
+      case e: ApiException => Future.successful(WrappedResponse(e.code, e.getMessage))
+      case _ => Future.successful(WrappedResponse(ErrorCodes.InternalServerError, exception.toString))
+    }
+  }
+}

app/controllers/EfController.scala

@@ -0,0 +1,80 @@
+package controllers
+
+import io.swagger.annotations.ApiParam
+import play.api.mvc._
+import protocol.WrappedResponse
+import repo.EfRepository
+import repo.models.{VolumeId, WorksetId}
+import utils.Helper.tokenize
+import utils.IdUtils._
+
+import javax.inject.Inject
+import scala.concurrent.ExecutionContext
+
+class EfController @Inject()(efRepository: EfRepository,
+                             components: ControllerComponents)
+                            (implicit val ec: ExecutionContext) extends AbstractController(components) {
+
+  def createWorkset(): Action[String] =
+    Action.async(parse.text) { implicit req =>
+      render.async {
+        case Accepts.Json() =>
+          val ids = tokenize(req.body, delims = " \n").toSet
+          efRepository.createWorkset(ids).map(WrappedResponse(_))
+      }
+    }
+
+  def getWorkset(@ApiParam(value = "the workset ID", required = true) wid: WorksetId): Action[AnyContent] =
+    Action.async { implicit req =>
+      render.async {
+        case Accepts.Json() =>
+          efRepository
+            .getWorkset(wid)
+            .map(WrappedResponse(_))
+      }
+    }
+
+  def deleteWorkset(@ApiParam(value = "the workset ID", required = true) wid: WorksetId): Action[AnyContent] =
+    Action.async { implicit req =>
+      render.async {
+        case Accepts.Json() =>
+          efRepository.deleteWorkset(wid).map(_ => WrappedResponse.Empty)
+      }
+    }
+
+  def getWorksetVolumesAggNoPos(@ApiParam(value = "the workset ID", required = true) wid: WorksetId,
+                                @ApiParam(value = "comma-separated list of fields to return") fields: Option[String]): Action[AnyContent] =
+    Action.async { implicit req =>
+      render.async {
+        case Accepts.Json() =>
+          efRepository
+            .getWorkset(wid)
+            .flatMap(workset => efRepository.getVolumesAggNoPos(workset.htids, fields.map(tokenize(_)).getOrElse(List.empty)))
+            .map(WrappedResponse(_))
+      }
+    }
+
+  def getWorksetAggNoPos(@ApiParam(value = "the workset ID", required = true) wid: WorksetId,
+                         @ApiParam(value = "comma-separated list of fields to return") fields: Option[String]): Action[AnyContent] =
+    Action.async { implicit req =>
+      render.async {
+        case Accepts.Json() =>
+          efRepository
+            .getWorkset(wid)
+            .flatMap(workset => efRepository.getWorksetAggNoPos(workset.htids, fields.map(tokenize(_)).getOrElse(List.empty)))
+            .map(WrappedResponse(_))
+      }
+    }
+
+  def getWorksetVolumesMetadata(@ApiParam(value = "the workset ID", required = true) wid: WorksetId,
+                                @ApiParam(value = "comma-separated list of fields to return") fields: Option[String]): Action[AnyContent] =
+    Action.async { implicit req =>
+      render.async {
+        case Accepts.Json() =>
+          efRepository
+            .getWorkset(wid)
+            .flatMap(workset => efRepository.getVolumesMetadata(workset.htids, fields.map(tokenize(_)).getOrElse(List.empty)))
+            .map(WrappedResponse(_))
+      }
+    }
+}

app/controllers/HealthController.scala

@@ -0,0 +1,36 @@
+package controllers
+
+import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents}
+import play.modules.reactivemongo.ReactiveMongoApi
+import reactivemongo.api.DB
+import reactivemongo.api.bson.{BSONDocument, BSONDocumentReader, BSONDocumentWriter}
+import utils.Ping
+
+import javax.inject._
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.Success
+
+@Singleton
+class HealthController @Inject()(reactiveMongoApi: ReactiveMongoApi, components: ControllerComponents)
+                                (implicit val ec: ExecutionContext) extends AbstractController(components) {
+
+  def livenessCheck: Action[AnyContent] = Action { Ok }
+
+  def readynessCheck: Action[AnyContent] =
+    Action.async { _ =>
+      reactiveMongoApi.database.flatMap(runPing).transform {
+        case Success(true) => Success(Ok)
+        case _ => Success(ServiceUnavailable)
+      }
+    }
+
+
+  implicit val pingCommandWriter: BSONDocumentWriter[Ping.type] =
+    BSONDocumentWriter[Ping.type] { _: Ping.type => BSONDocument("ping" -> 1) }
+
+  implicit val pingResultReader: BSONDocumentReader[Boolean] =
+    BSONDocumentReader.option[Boolean] { _.booleanLike("ok") }
+
+  private def runPing(db: DB)(implicit ec: ExecutionContext): Future[Boolean] = db.runCommand(Ping)
+
+}

app/exceptions/ApiException.scala

@@ -0,0 +1,9 @@
+package exceptions
+
+class ApiException(message: String, val code: Int = ErrorCodes.GenericApplicationError, cause: Throwable = null)
+  extends Exception(message, cause)
+
+object ApiException {
+  def apply(message: String, code: Int = ErrorCodes.GenericApplicationError, cause: Throwable = null): ApiException =
+    new ApiException(message, code, cause)
+}

app/exceptions/BadRequestException.scala

@@ -0,0 +1,9 @@
+package exceptions
+
+class BadRequestException(message: String, cause: Throwable = null)
+  extends ApiException(message, ErrorCodes.BadRequest, cause)
+
+object BadRequestException {
+  def apply(message: String, cause: Throwable = null): BadRequestException =
+    new BadRequestException(message, cause)
+}