diff --git a/app/search/docs/[...pkg]/search.tsx b/app/search/docs/[...pkg]/search.tsx index 76a30d4..650c058 100644 --- a/app/search/docs/[...pkg]/search.tsx +++ b/app/search/docs/[...pkg]/search.tsx @@ -2,7 +2,10 @@ import { useEffect, useState } from "react"; import styles from "./search.module.scss"; -import { getPackageDocs } from "../../../../client/api/get-package-docs"; +import { + getPackageDocs, + getPackageDocsSync, +} from "../../../../client/api/get-package-docs"; import Placeholder from "../../../../client/components/Placeholder"; import Header from "../../../../client/components/Header"; import Footer from "../../../../client/components/Footer"; @@ -42,7 +45,7 @@ export default function Search({ pkg }) { const searchAndRedirect = async (pkg: string, version: string | null) => { try { - const result = await getPackageDocs(pkg, version, { force }); + const result = await getPackageDocsSync(pkg, version, { force }); if (result.status === "success") { window.location.href = `/docs/${ diff --git a/client/api/get-package-docs.ts b/client/api/get-package-docs.ts index 563ab9d..c81840e 100644 --- a/client/api/get-package-docs.ts +++ b/client/api/get-package-docs.ts @@ -16,6 +16,17 @@ type TriggerAPIResponse = pollInterval: number; }; +type BuildAPIResponse = + | { + status: "success"; + } + | { + status: "failed"; + errorCode: string; + errorMessage: string; + errorStack: string; + }; + type PollAPIResponse = | { status: "success" | "queued"; @@ -206,3 +217,54 @@ export async function getPackageDocs( "Failed because the polling API returned an unknown status: " + status, }; } + +export async function getPackageDocsSync( + pkg: string, + pkgVersion: string, + { force }: { force: boolean }, +): Promise { + let triggerResponse: AxiosResponse = null; + const withForce = force ? "?force=true" : ""; + + try { + triggerResponse = await axios.post( + `/api/docs/build/${ + pkgVersion ? [pkg, pkgVersion].join("/") : pkg + }${withForce}`, + ); + } catch (err) { + let errorMessage = ""; + if (err.response?.data) { + return { + status: "failure", + errorCode: err.response.data.name, + errorMessage: getErrorMessage(err.response.data), + }; + } + + return { + status: "failure", + errorCode: "UNEXPECTED_DOCS_TRIGGER_FAILURE", + errorMessage: errorMessage, + }; + } + + let triggerResult = triggerResponse.data; + const { status } = triggerResult; + + if (status === "success") { + return { + status: "success", + }; + } + + return { + status: "failure", + errorCode: triggerResult.errorCode, + errorMessage: getErrorMessage({ + name: triggerResult.errorCode, + extra: triggerResult.errorMessage, + errorStack: triggerResult.errorStack, + }), + }; +} diff --git a/package.json b/package.json index 9ee886d..42758a2 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ "@types/validate-npm-package-name": "^4.0.2", "@vitejs/plugin-react": "^4.2.1", "animejs": "^3.2.2", - "axios": "^1.6.3", + "axios": "^1.6.4", "bullmq": "patch:bullmq@npm%3A4.15.4#~/.yarn/patches/bullmq-npm-4.15.4-b55917dd70.patch", "chalk": "^4.1.2", "check-disk-space": "^3.4.0", diff --git a/server.ts b/server.ts index d5d5403..209f5aa 100644 --- a/server.ts +++ b/server.ts @@ -3,6 +3,7 @@ import fastifyStart from "fastify"; import fastifyStatic from "@fastify/static"; import next from "next"; import { + handlerAPIDocsBuild, handlerAPIDocsPoll, handlerAPIDocsTrigger, handlerDocsHTML, @@ -146,6 +147,12 @@ app handler: handlerAPIDocsTrigger, }); + fastify.route({ + method: "POST", + url: `/api/docs/build/*`, + handler: handlerAPIDocsBuild, + }); + fastify.setErrorHandler(function (error, request, reply) { logger.error(error); diff --git a/server/package/index.ts b/server/package/index.ts index beb52dc..fbc63db 100644 --- a/server/package/index.ts +++ b/server/package/index.ts @@ -11,6 +11,19 @@ import { parse } from "node-html-parser"; import fs from "fs"; import * as stackTraceParser from "stacktrace-parser"; import { LRUCache } from "lru-cache"; +import workerpool from "workerpool"; + +const docsWorkerPool = workerpool.pool( + path.join(__dirname, "..", "workers", "docs-builder-worker-pool.js"), + { + workerType: "process", + maxQueueSize: 10, + forkOpts: { + stdio: "inherit", + }, + forkArgs: ["--max-old-space-size=1024"], + }, +); export async function resolveDocsRequest({ packageName, @@ -146,6 +159,55 @@ export async function handlerAPIDocsTrigger(req, res) { } } +export async function handlerAPIDocsBuild(req, res) { + const paramsPath = req.params["*"]; + const { force } = req.query; + const routePackageDetails = packageFromPath(paramsPath); + logger.info("routePackageDetails is ", routePackageDetails); + + if (!routePackageDetails) { + logger.error("Route package details not found in " + paramsPath); + return res.status(404).send({ + name: PackageNotFoundError.name, + }); + } + + const { packageName, packageVersion, docsFragment } = routePackageDetails; + + const resolvedRequest = await resolveDocsRequest({ + packageName, + packageVersion, + force, + }); + + if (resolvedRequest.type === "hit") { + return res.send({ status: "success" }); + } else { + logger.info( + 'Docs job for "%s" at version %s queued for building', + resolvedRequest.packageName, + resolvedRequest.packageVersion, + ); + + try { + await docsWorkerPool.exec("generateDocs", [ + { packageJSON: resolvedRequest.packageJSON, force }, + ]); + + return res.send({ status: "success" }); + } catch (err) { + return res.send({ + status: "failed", + errorCode: err.message, + errorMessage: err.originalError?.message, + errorStack: + cleanStackTrace(err.originalError?.stacktrace) || + err.originalError?.message, + }); + } + } +} + function cleanStackTrace(stackTrace: string | undefined) { if (!stackTrace) return ""; diff --git a/server/workers/docs-builder-worker-pool.js b/server/workers/docs-builder-worker-pool.js new file mode 100644 index 0000000..5808ac7 --- /dev/null +++ b/server/workers/docs-builder-worker-pool.js @@ -0,0 +1,16 @@ +require("esbuild-register/dist/node").register(); + +const { + generateDocsForPackage, +} = require("../package/extractor/doc-generator"); +const workerpool = require("workerpool"); + +async function generateDocs(job) { + return await generateDocsForPackage(job.packageJSON, { + force: job.force, + }); +} + +workerpool.worker({ + generateDocs, +}); diff --git a/yarn.lock b/yarn.lock index 57327db..461981c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2235,14 +2235,14 @@ __metadata: languageName: node linkType: hard -"axios@npm:^1.6.3": - version: 1.6.3 - resolution: "axios@npm:1.6.3" +"axios@npm:^1.6.4": + version: 1.6.5 + resolution: "axios@npm:1.6.5" dependencies: - follow-redirects: "npm:^1.15.0" + follow-redirects: "npm:^1.15.4" form-data: "npm:^4.0.0" proxy-from-env: "npm:^1.1.0" - checksum: 78e72ec40ee49b85f076758e65d04dd966361cbf82f3789e53c5a1b6814ef57825098f8a997ec1846cf004fb7407655c7b96c496d8133d88ca0f73a98a9c452b + checksum: 465489d9bf8f039b9adbc8103b6299d6a5e26de77b27f0e4173d814d39bca8f4b4659d94e09ee40461aedccd8c2452f1e2b3edace1c9f81220060d2974ff9dc7 languageName: node linkType: hard @@ -3833,13 +3833,13 @@ __metadata: languageName: node linkType: hard -"follow-redirects@npm:^1.14.0, follow-redirects@npm:^1.15.0": - version: 1.15.3 - resolution: "follow-redirects@npm:1.15.3" +"follow-redirects@npm:^1.14.0, follow-redirects@npm:^1.15.4": + version: 1.15.4 + resolution: "follow-redirects@npm:1.15.4" peerDependenciesMeta: debug: optional: true - checksum: 60d98693f4976892f8c654b16ef6d1803887a951898857ab0cdc009570b1c06314ad499505b7a040ac5b98144939f8597766e5e6a6859c0945d157b473aa6f5f + checksum: 2e8f5f259a6b02dfa8dc199e08431848a7c3beed32eb4c19945966164a52c89f07b86c3afcc32ebe4279cf0a960520e45a63013d6350309c5ec90133c5d9351a languageName: node linkType: hard @@ -7970,7 +7970,7 @@ __metadata: "@types/validate-npm-package-name": "npm:^4.0.2" "@vitejs/plugin-react": "npm:^4.2.1" animejs: "npm:^3.2.2" - axios: "npm:^1.6.3" + axios: "npm:^1.6.4" bullmq: "patch:bullmq@npm%3A4.15.4#~/.yarn/patches/bullmq-npm-4.15.4-b55917dd70.patch" chalk: "npm:^4.1.2" check-disk-space: "npm:^3.4.0"