Merge branch 'pastelsky:main' into main

app/search/docs/[...pkg]/search.tsx

@@ -2,7 +2,10 @@
 
 import { useEffect, useState } from "react";
 import styles from "./search.module.scss";
-import { getPackageDocs } from "../../../../client/api/get-package-docs";
+import {
+  getPackageDocs,
+  getPackageDocsSync,
+} from "../../../../client/api/get-package-docs";
 import Placeholder from "../../../../client/components/Placeholder";
 import Header from "../../../../client/components/Header";
 import Footer from "../../../../client/components/Footer";
@@ -42,7 +45,7 @@ export default function Search({ pkg }) {
 
   const searchAndRedirect = async (pkg: string, version: string | null) => {
     try {
-      const result = await getPackageDocs(pkg, version, { force });
+      const result = await getPackageDocsSync(pkg, version, { force });
 
       if (result.status === "success") {
         window.location.href = `/docs/${

client/api/get-package-docs.ts

@@ -16,6 +16,17 @@ type TriggerAPIResponse =
       pollInterval: number;
     };
 
+type BuildAPIResponse =
+  | {
+      status: "success";
+    }
+  | {
+      status: "failed";
+      errorCode: string;
+      errorMessage: string;
+      errorStack: string;
+    };
+
 type PollAPIResponse =
   | {
       status: "success" | "queued";
@@ -206,3 +217,54 @@ export async function getPackageDocs(
       "Failed because the polling API returned an unknown status: " + status,
   };
 }
+
+export async function getPackageDocsSync(
+  pkg: string,
+  pkgVersion: string,
+  { force }: { force: boolean },
+): Promise<PackageDocsResponse> {
+  let triggerResponse: AxiosResponse<BuildAPIResponse> = null;
+  const withForce = force ? "?force=true" : "";
+
+  try {
+    triggerResponse = await axios.post(
+      `/api/docs/build/${
+        pkgVersion ? [pkg, pkgVersion].join("/") : pkg
+      }${withForce}`,
+    );
+  } catch (err) {
+    let errorMessage = "";
+    if (err.response?.data) {
+      return {
+        status: "failure",
+        errorCode: err.response.data.name,
+        errorMessage: getErrorMessage(err.response.data),
+      };
+    }
+
+    return {
+      status: "failure",
+      errorCode: "UNEXPECTED_DOCS_TRIGGER_FAILURE",
+      errorMessage: errorMessage,
+    };
+  }
+
+  let triggerResult = triggerResponse.data;
+  const { status } = triggerResult;
+
+  if (status === "success") {
+    return {
+      status: "success",
+    };
+  }
+
+  return {
+    status: "failure",
+    errorCode: triggerResult.errorCode,
+    errorMessage: getErrorMessage({
+      name: triggerResult.errorCode,
+      extra: triggerResult.errorMessage,
+      errorStack: triggerResult.errorStack,
+    }),
+  };
+}

package.json

@@ -26,7 +26,7 @@
     "@types/validate-npm-package-name": "^4.0.2",
     "@vitejs/plugin-react": "^4.2.1",
     "animejs": "^3.2.2",
-    "axios": "^1.6.3",
+    "axios": "^1.6.4",
     "bullmq": "patch:bullmq@npm%3A4.15.4#~/.yarn/patches/bullmq-npm-4.15.4-b55917dd70.patch",
     "chalk": "^4.1.2",
     "check-disk-space": "^3.4.0",

server.ts

@@ -3,6 +3,7 @@ import fastifyStart from "fastify";
 import fastifyStatic from "@fastify/static";
 import next from "next";
 import {
+  handlerAPIDocsBuild,
   handlerAPIDocsPoll,
   handlerAPIDocsTrigger,
   handlerDocsHTML,
@@ -146,6 +147,12 @@ app
       handler: handlerAPIDocsTrigger,
     });
 
+    fastify.route({
+      method: "POST",
+      url: `/api/docs/build/*`,
+      handler: handlerAPIDocsBuild,
+    });
+
     fastify.setErrorHandler(function (error, request, reply) {
       logger.error(error);
 

server/package/index.ts

@@ -11,6 +11,19 @@ import { parse } from "node-html-parser";
 import fs from "fs";
 import * as stackTraceParser from "stacktrace-parser";
 import { LRUCache } from "lru-cache";
+import workerpool from "workerpool";
+
+const docsWorkerPool = workerpool.pool(
+  path.join(__dirname, "..", "workers", "docs-builder-worker-pool.js"),
+  {
+    workerType: "process",
+    maxQueueSize: 10,
+    forkOpts: {
+      stdio: "inherit",
+    },
+    forkArgs: ["--max-old-space-size=1024"],
+  },
+);
 
 export async function resolveDocsRequest({
   packageName,
@@ -146,6 +159,55 @@ export async function handlerAPIDocsTrigger(req, res) {
   }
 }
 
+export async function handlerAPIDocsBuild(req, res) {
+  const paramsPath = req.params["*"];
+  const { force } = req.query;
+  const routePackageDetails = packageFromPath(paramsPath);
+  logger.info("routePackageDetails is ", routePackageDetails);
+
+  if (!routePackageDetails) {
+    logger.error("Route package details not found in " + paramsPath);
+    return res.status(404).send({
+      name: PackageNotFoundError.name,
+    });
+  }
+
+  const { packageName, packageVersion, docsFragment } = routePackageDetails;
+
+  const resolvedRequest = await resolveDocsRequest({
+    packageName,
+    packageVersion,
+    force,
+  });
+
+  if (resolvedRequest.type === "hit") {
+    return res.send({ status: "success" });
+  } else {
+    logger.info(
+      'Docs job for "%s" at version %s queued for building',
+      resolvedRequest.packageName,
+      resolvedRequest.packageVersion,
+    );
+
+    try {
+      await docsWorkerPool.exec("generateDocs", [
+        { packageJSON: resolvedRequest.packageJSON, force },
+      ]);
+
+      return res.send({ status: "success" });
+    } catch (err) {
+      return res.send({
+        status: "failed",
+        errorCode: err.message,
+        errorMessage: err.originalError?.message,
+        errorStack:
+          cleanStackTrace(err.originalError?.stacktrace) ||
+          err.originalError?.message,
+      });
+    }
+  }
+}
+
 function cleanStackTrace(stackTrace: string | undefined) {
   if (!stackTrace) return "";
 

server/workers/docs-builder-worker-pool.js

@@ -0,0 +1,16 @@
+require("esbuild-register/dist/node").register();
+
+const {
+  generateDocsForPackage,
+} = require("../package/extractor/doc-generator");
+const workerpool = require("workerpool");
+
+async function generateDocs(job) {
+  return await generateDocsForPackage(job.packageJSON, {
+    force: job.force,
+  });
+}
+
+workerpool.worker({
+  generateDocs,
+});

yarn.lock

@@ -2235,14 +2235,14 @@ __metadata:
   languageName: node
   linkType: hard
 
-"axios@npm:^1.6.3":
-  version: 1.6.3
-  resolution: "axios@npm:1.6.3"
+"axios@npm:^1.6.4":
+  version: 1.6.5
+  resolution: "axios@npm:1.6.5"
   dependencies:
-    follow-redirects: "npm:^1.15.0"
+    follow-redirects: "npm:^1.15.4"
     form-data: "npm:^4.0.0"
     proxy-from-env: "npm:^1.1.0"
-  checksum: 78e72ec40ee49b85f076758e65d04dd966361cbf82f3789e53c5a1b6814ef57825098f8a997ec1846cf004fb7407655c7b96c496d8133d88ca0f73a98a9c452b
+  checksum: 465489d9bf8f039b9adbc8103b6299d6a5e26de77b27f0e4173d814d39bca8f4b4659d94e09ee40461aedccd8c2452f1e2b3edace1c9f81220060d2974ff9dc7
   languageName: node
   linkType: hard
 
@@ -3833,13 +3833,13 @@ __metadata:
   languageName: node
   linkType: hard
 
-"follow-redirects@npm:^1.14.0, follow-redirects@npm:^1.15.0":
-  version: 1.15.3
-  resolution: "follow-redirects@npm:1.15.3"
+"follow-redirects@npm:^1.14.0, follow-redirects@npm:^1.15.4":
+  version: 1.15.4
+  resolution: "follow-redirects@npm:1.15.4"
   peerDependenciesMeta:
     debug:
       optional: true
-  checksum: 60d98693f4976892f8c654b16ef6d1803887a951898857ab0cdc009570b1c06314ad499505b7a040ac5b98144939f8597766e5e6a6859c0945d157b473aa6f5f
+  checksum: 2e8f5f259a6b02dfa8dc199e08431848a7c3beed32eb4c19945966164a52c89f07b86c3afcc32ebe4279cf0a960520e45a63013d6350309c5ec90133c5d9351a
   languageName: node
   linkType: hard
 
@@ -7970,7 +7970,7 @@ __metadata:
     "@types/validate-npm-package-name": "npm:^4.0.2"
     "@vitejs/plugin-react": "npm:^4.2.1"
     animejs: "npm:^3.2.2"
-    axios: "npm:^1.6.3"
+    axios: "npm:^1.6.4"
     bullmq: "patch:bullmq@npm%3A4.15.4#~/.yarn/patches/bullmq-npm-4.15.4-b55917dd70.patch"
     chalk: "npm:^4.1.2"
     check-disk-space: "npm:^3.4.0"