diff --git a/README.md b/README.md index 83035ea3..93088c79 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ Options: -t, --target name of schema to use (choices: "allowed-amounts", "in-network-rates", "provider-reference", "table-of-contents", default: "in-network-rates") -s, --strict enable strict checking, which prohibits additional properties in data file + -y, --yes-all automatically respond "yes" to confirmation prompts -h, --help display help for command ``` @@ -151,6 +152,7 @@ Options: -t, --target name of schema to use (choices: "allowed-amounts", "in-network-rates", "provider-reference", "table-of-contents", default: "in-network-rates") -s, --strict enable strict checking, which prohibits additional properties in data file + -y, --yes-all automatically respond "yes" to confirmation prompts -h, --help display help for command ``` diff --git a/src/DockerManager.ts b/src/DockerManager.ts index cb2f1a52..fcbb175a 100644 --- a/src/DockerManager.ts +++ b/src/DockerManager.ts @@ -8,7 +8,7 @@ import { logger } from './logger'; export class DockerManager { containerId = ''; - constructor() {} + constructor(public outputPath = '') {} private async initContainerId(): Promise { this.containerId = await util @@ -24,7 +24,7 @@ export class DockerManager { schemaPath: string, schemaName: string, dataPath: string, - outputPath: string + outputPath = this.outputPath ): Promise { try { if (this.containerId.length === 0) { @@ -42,11 +42,8 @@ export class DockerManager { logger.debug(runCommand); return util .promisify(exec)(runCommand) - .then(result => { + .then(() => { const containerResult: ContainerResult = { pass: true }; - if (outputPath && fs.existsSync(containerOutputPath)) { - fs.copySync(containerOutputPath, outputPath); - } if (fs.existsSync(containerOutputPath)) { if (outputPath) { fs.copySync(containerOutputPath, outputPath); @@ -64,10 +61,7 @@ export class DockerManager { } return containerResult; }) - .catch(reason => { - if (outputPath && fs.existsSync(path.join(outputDir, 'output.txt'))) { - fs.copySync(path.join(outputDir, 'output.txt'), outputPath); - } + .catch(() => { if (fs.existsSync(containerOutputPath)) { if (outputPath) { fs.copySync(containerOutputPath, outputPath); @@ -115,6 +109,7 @@ export class DockerManager { export type ContainerResult = { pass: boolean; + text?: string; locations?: { inNetwork?: string[]; allowedAmount?: string[]; diff --git a/src/DownloadManager.ts b/src/DownloadManager.ts new file mode 100644 index 00000000..8b957fbc --- /dev/null +++ b/src/DownloadManager.ts @@ -0,0 +1,162 @@ +import axios from 'axios'; +import readlineSync from 'readline-sync'; +import fs from 'fs-extra'; +import temp from 'temp'; +import path from 'path'; +import yauzl from 'yauzl'; +import { createGunzip } from 'zlib'; +import { ZipContents, isGzip, isZip } from './utils'; +import { logger } from './logger'; + +import { pipeline } from 'stream/promises'; + +const ONE_MEGABYTE = 1024 * 1024; +const DATA_SIZE_WARNING_THRESHOLD = ONE_MEGABYTE * 1024; // 1 gigabyte + +export class DownloadManager { + folder: string; + + constructor(public alwaysYes = false) { + temp.track(); + this.folder = temp.mkdirSync(); + } + + async checkDataUrl(url: string): Promise { + try { + const response = await axios.head(url); + if (response.status === 200) { + let proceedToDownload: boolean; + const contentLength = parseInt(response.headers['content-length']); + if (this.alwaysYes) { + proceedToDownload = true; + } else if (isNaN(contentLength)) { + proceedToDownload = readlineSync.keyInYNStrict( + 'Data file size is unknown. Download this file?' + ); + } else if (contentLength > DATA_SIZE_WARNING_THRESHOLD) { + proceedToDownload = readlineSync.keyInYNStrict( + `Data file is ${(contentLength / ONE_MEGABYTE).toFixed( + 2 + )} MB in size. Download this file?` + ); + } else { + proceedToDownload = true; + } + return proceedToDownload; + } else { + logger.error( + `Received unsuccessful status code ${response.status} when checking data file URL: ${url}` + ); + return false; + } + } catch (e) { + logger.error(`Request failed when checking data file URL: ${url}`); + logger.error(e.message); + return false; + } + } + + async downloadDataFile(url: string, folder = this.folder): Promise { + const filenameGuess = 'data.json'; + const dataPath = path.join(folder, filenameGuess); + return new Promise((resolve, reject) => { + logger.info('Beginning download...\n'); + axios({ + method: 'get', + url: url, + responseType: 'stream', + onDownloadProgress: progressEvent => { + if (process.stdout.isTTY) { + let progressText: string; + if (progressEvent.progress != null) { + progressText = `Downloaded ${Math.floor(progressEvent.progress * 100)}% of file (${ + progressEvent.loaded + } bytes)`; + } else { + progressText = `Downloaded ${progressEvent.loaded} bytes`; + } + process.stdout.clearLine(0, () => { + process.stdout.cursorTo(0, () => { + process.stdout.moveCursor(0, -1, () => { + logger.info(progressText); + }); + }); + }); + } + } + }) + .then(response => { + const contentType = response.headers['content-type'] ?? 'application/octet-stream'; + const finalUrl = response.request.path; + if (isZip(contentType, finalUrl)) { + // zips require additional work to find a JSON file inside + const zipPath = path.join(folder, 'data.zip'); + const zipOutputStream = fs.createWriteStream(zipPath); + pipeline(response.data, zipOutputStream).then(() => { + yauzl.open(zipPath, { lazyEntries: true, autoClose: false }, (err, zipFile) => { + if (err != null) { + reject(err); + } + const jsonEntries: yauzl.Entry[] = []; + + zipFile.on('entry', (entry: yauzl.Entry) => { + if (entry.fileName.endsWith('.json')) { + jsonEntries.push(entry); + } + zipFile.readEntry(); + }); + + zipFile.on('end', () => { + logger.info('Download complete.'); + if (jsonEntries.length === 0) { + reject('No JSON file present in zip.'); + } else { + let chosenEntry: yauzl.Entry; + if (jsonEntries.length === 1) { + chosenEntry = jsonEntries[0]; + zipFile.openReadStream(chosenEntry, (err, readStream) => { + const outputStream = fs.createWriteStream(dataPath); + outputStream.on('finish', () => { + zipFile.close(); + resolve(dataPath); + }); + outputStream.on('error', () => { + zipFile.close(); + reject('Error writing downloaded file.'); + }); + readStream.pipe(outputStream); + }); + } else { + jsonEntries.sort((a, b) => { + return a.fileName.localeCompare(b.fileName); + }); + resolve({ zipFile, jsonEntries, dataPath }); + } + } + }); + zipFile.readEntry(); + }); + }); + } else { + const outputStream = fs.createWriteStream(dataPath); + outputStream.on('finish', () => { + logger.info('Download complete.'); + resolve(dataPath); + }); + outputStream.on('error', () => { + reject('Error writing downloaded file.'); + }); + + if (isGzip(contentType, finalUrl)) { + pipeline(response.data, createGunzip(), outputStream); + } else { + response.data.pipe(outputStream); + } + } + }) + .catch(reason => { + reject('Error downloading data file.'); + }); + }); + } +} diff --git a/src/commands.ts b/src/commands.ts index ce33c151..c6414197 100644 --- a/src/commands.ts +++ b/src/commands.ts @@ -7,8 +7,6 @@ import { OptionValues } from 'commander'; import { config, - downloadDataFile, - checkDataUrl, chooseJsonFile, getEntryFromZip, assessTocContents, @@ -18,6 +16,7 @@ import temp from 'temp'; import { SchemaManager } from './SchemaManager'; import { DockerManager } from './DockerManager'; import { logger } from './logger'; +import { DownloadManager } from './DownloadManager'; export async function validate(dataFile: string, options: OptionValues) { // check to see if supplied json file exists @@ -61,12 +60,12 @@ export async function validate(dataFile: string, options: OptionValues) { .then(async schemaPath => { temp.track(); if (schemaPath != null) { - const dockerManager = new DockerManager(); + const dockerManager = new DockerManager(options.out); + const downloadManager = new DownloadManager(options.yesAll); const containerResult = await dockerManager.runContainer( schemaPath, options.target, - dataFile, - options.out + dataFile ); if (containerResult.pass) { if (options.target === 'table-of-contents') { @@ -74,13 +73,13 @@ export async function validate(dataFile: string, options: OptionValues) { containerResult.locations, schemaManager, dockerManager, - options.out + downloadManager ); await assessReferencedProviders( providerReferences, schemaManager, dockerManager, - options.out + downloadManager ); } else if ( options.target === 'in-network-rates' && @@ -90,7 +89,7 @@ export async function validate(dataFile: string, options: OptionValues) { containerResult.locations.providerReference, schemaManager, dockerManager, - options.out + downloadManager ); } } @@ -108,7 +107,8 @@ export async function validate(dataFile: string, options: OptionValues) { export async function validateFromUrl(dataUrl: string, options: OptionValues) { temp.track(); - if (await checkDataUrl(dataUrl)) { + const downloadManager = new DownloadManager(options.yesAll); + if (await downloadManager.checkDataUrl(dataUrl)) { const schemaManager = new SchemaManager(); await schemaManager.ensureRepo(); schemaManager.strict = options.strict; @@ -121,14 +121,13 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) { }) .then(async schemaPath => { if (schemaPath != null) { - const dockerManager = new DockerManager(); - const dataFile = await downloadDataFile(dataUrl, temp.mkdirSync()); + const dockerManager = new DockerManager(options.out); + const dataFile = await downloadManager.downloadDataFile(dataUrl); if (typeof dataFile === 'string') { const containerResult = await dockerManager.runContainer( schemaPath, options.target, - dataFile, - options.out + dataFile ); if (containerResult.pass) { if (options.target === 'table-of-contents') { @@ -136,13 +135,13 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) { containerResult.locations, schemaManager, dockerManager, - options.out + downloadManager ); await assessReferencedProviders( providerReferences, schemaManager, dockerManager, - options.out + downloadManager ); } else if ( options.target === 'in-network-rates' && @@ -152,7 +151,7 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) { containerResult.locations.providerReference, schemaManager, dockerManager, - options.out + downloadManager ); } } @@ -163,12 +162,7 @@ export async function validateFromUrl(dataUrl: string, options: OptionValues) { while (continuation === true) { const chosenEntry = chooseJsonFile(dataFile.jsonEntries); await getEntryFromZip(dataFile.zipFile, chosenEntry, dataFile.dataPath); - await dockerManager.runContainer( - schemaPath, - options.target, - dataFile.dataPath, - options.out - ); + await dockerManager.runContainer(schemaPath, options.target, dataFile.dataPath); continuation = readlineSync.keyInYNStrict( 'Would you like to validate another file in the ZIP?' ); diff --git a/src/index.ts b/src/index.ts index 0cc8fdd1..a1493a5a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -14,7 +14,7 @@ async function main() { .name('cms-mrf-validator') .description('Tool for validating health coverage machine-readable files.') .option('-d, --debug', 'show debug output') - .hook('preAction', (thisCommand, actionCommand) => { + .hook('preAction', thisCommand => { if (thisCommand.opts().debug) { logger.level = 'debug'; logger.debug(process.argv.join(' ')); @@ -35,6 +35,7 @@ async function main() { '-s, --strict', 'enable strict checking, which prohibits additional properties in data file' ) + .option('-y, --yes-all', 'automatically respond "yes" to confirmation prompts') .action(validate); program @@ -55,6 +56,7 @@ async function main() { '-s, --strict', 'enable strict checking, which prohibits additional properties in data file' ) + .option('-y, --yes-all', 'automatically respond "yes" to confirmation prompts') .action((dataUrl, options) => { validateFromUrl(dataUrl, options).then(result => { if (result) { diff --git a/src/utils.ts b/src/utils.ts index ee0aa51a..afd8ea3c 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,17 +1,13 @@ -import util from 'util'; -import axios from 'axios'; import readlineSync from 'readline-sync'; import path from 'path'; -import { exec } from 'child_process'; import fs from 'fs-extra'; import temp from 'temp'; -import { createGunzip } from 'zlib'; -import { pipeline } from 'stream/promises'; import yauzl from 'yauzl'; import { EOL } from 'os'; import { DockerManager } from './DockerManager'; import { SchemaManager } from './SchemaManager'; import { logger } from './logger'; +import { DownloadManager } from './DownloadManager'; export type ZipContents = { zipFile: yauzl.ZipFile; @@ -30,16 +26,6 @@ export const config = { SCHEMA_REPO_FOLDER: path.normalize(path.join(__dirname, '..', 'schema-repo')) }; -const ONE_MEGABYTE = 1024 * 1024; -const DATA_SIZE_WARNING_THRESHOLD = ONE_MEGABYTE * 1024; // 1 gigabyte - -export async function ensureRepo(repoDirectory: string) { - // check if the repo exists, and if not, try to clone it - if (!fs.existsSync(path.join(repoDirectory, '.git'))) { - return util.promisify(exec)(`git clone ${config.SCHEMA_REPO_URL} "${repoDirectory}"`); - } -} - type ContainerResult = { pass: boolean; locations?: { @@ -49,143 +35,6 @@ type ContainerResult = { }; }; -export async function checkDataUrl(url: string) { - try { - const response = await axios.head(url); - if (response.status === 200) { - let proceedToDownload: boolean; - const contentLength = parseInt(response.headers['content-length']); - if (isNaN(contentLength)) { - proceedToDownload = readlineSync.keyInYNStrict( - 'Data file size is unknown. Download this file?' - ); - } else if (contentLength > DATA_SIZE_WARNING_THRESHOLD) { - proceedToDownload = readlineSync.keyInYNStrict( - `Data file is ${(contentLength / ONE_MEGABYTE).toFixed( - 2 - )} MB in size. Download this file?` - ); - } else { - proceedToDownload = true; - } - return proceedToDownload; - } else { - logger.error( - `Received unsuccessful status code ${response.status} when checking data file URL: ${url}` - ); - return false; - } - } catch (e) { - logger.error(`Request failed when checking data file URL: ${url}`); - logger.error(e.message); - return false; - } -} - -export async function downloadDataFile(url: string, folder: string): Promise { - const filenameGuess = 'data.json'; - const dataPath = path.join(folder, filenameGuess); - return new Promise((resolve, reject) => { - logger.info('Beginning download...\n'); - axios({ - method: 'get', - url: url, - responseType: 'stream', - onDownloadProgress: progressEvent => { - if (process.stdout.isTTY) { - let progressText: string; - if (progressEvent.progress != null) { - progressText = `Downloaded ${Math.floor(progressEvent.progress * 100)}% of file (${ - progressEvent.loaded - } bytes)`; - } else { - progressText = `Downloaded ${progressEvent.loaded} bytes`; - } - process.stdout.clearLine(0, () => { - process.stdout.cursorTo(0, () => { - process.stdout.moveCursor(0, -1, () => { - logger.info(progressText); - }); - }); - }); - } - } - }) - .then(response => { - const contentType = response.headers['content-type'] ?? 'application/octet-stream'; - const finalUrl = response.request.path; - if (isZip(contentType, finalUrl)) { - // zips require additional work to find a JSON file inside - const zipPath = path.join(folder, 'data.zip'); - const zipOutputStream = fs.createWriteStream(zipPath); - pipeline(response.data, zipOutputStream).then(() => { - yauzl.open(zipPath, { lazyEntries: true, autoClose: false }, (err, zipFile) => { - if (err != null) { - reject(err); - } - const jsonEntries: yauzl.Entry[] = []; - - zipFile.on('entry', (entry: yauzl.Entry) => { - if (entry.fileName.endsWith('.json')) { - jsonEntries.push(entry); - } - zipFile.readEntry(); - }); - - zipFile.on('end', () => { - logger.info('Download complete.'); - if (jsonEntries.length === 0) { - reject('No JSON file present in zip.'); - } else { - let chosenEntry: yauzl.Entry; - if (jsonEntries.length === 1) { - chosenEntry = jsonEntries[0]; - zipFile.openReadStream(chosenEntry, (err, readStream) => { - const outputStream = fs.createWriteStream(dataPath); - outputStream.on('finish', () => { - zipFile.close(); - resolve(dataPath); - }); - outputStream.on('error', () => { - zipFile.close(); - reject('Error writing downloaded file.'); - }); - readStream.pipe(outputStream); - }); - } else { - jsonEntries.sort((a, b) => { - return a.fileName.localeCompare(b.fileName); - }); - resolve({ zipFile, jsonEntries, dataPath }); - } - } - }); - zipFile.readEntry(); - }); - }); - } else { - const outputStream = fs.createWriteStream(dataPath); - outputStream.on('finish', () => { - logger.info('Download complete.'); - resolve(dataPath); - }); - outputStream.on('error', () => { - reject('Error writing downloaded file.'); - }); - - if (isGzip(contentType, finalUrl)) { - pipeline(response.data, createGunzip(), outputStream); - } else { - response.data.pipe(outputStream); - } - } - }) - .catch(reason => { - reject('Error downloading data file.'); - }); - }); -} - export async function getEntryFromZip( zipFile: yauzl.ZipFile, entry: yauzl.Entry, @@ -211,7 +60,7 @@ export async function getEntryFromZip( }); } -function isGzip(contentType: string, url: string): boolean { +export function isGzip(contentType: string, url: string): boolean { return ( contentType === 'application/gzip' || contentType === 'application/x-gzip' || @@ -219,7 +68,7 @@ function isGzip(contentType: string, url: string): boolean { ); } -function isZip(contentType: string, url: string): boolean { +export function isZip(contentType: string, url: string): boolean { return ( contentType === 'application/zip' || (contentType === 'application/octet-stream' && /\.zip(\?|$)/.test(url)) @@ -305,7 +154,7 @@ export async function assessTocContents( locations: ContainerResult['locations'], schemaManager: SchemaManager, dockerManager: DockerManager, - outputPath: string + downloadManager: DownloadManager ): Promise { const totalFileCount = (locations?.inNetwork?.length ?? 0) + (locations?.allowedAmount?.length ?? 0); @@ -320,16 +169,16 @@ export async function assessTocContents( logger.info('== Allowed Amounts =='); locations.allowedAmount.forEach(aaf => logger.info(`* ${aaf}`)); } - const wantToValidateContents = readlineSync.keyInYNStrict( - `Would you like to validate ${fileText}?` - ); + const wantToValidateContents = + downloadManager.alwaysYes || + readlineSync.keyInYNStrict(`Would you like to validate ${fileText}?`); if (wantToValidateContents) { const providerReferences = await validateTocContents( locations.inNetwork ?? [], locations.allowedAmount ?? [], schemaManager, dockerManager, - outputPath + downloadManager ); return providerReferences; } @@ -342,11 +191,11 @@ export async function validateTocContents( allowedAmount: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - outputPath: string + downloadManager: DownloadManager ): Promise { temp.track(); let tempOutput = ''; - if (outputPath?.length > 0) { + if (dockerManager.outputPath?.length > 0) { tempOutput = path.join(temp.mkdirSync('contents'), 'contained-result'); } let providerReferences: Set; @@ -356,7 +205,7 @@ export async function validateTocContents( inNetwork, schemaManager, dockerManager, - outputPath, + downloadManager, tempOutput ); } else { @@ -364,7 +213,7 @@ export async function validateTocContents( inNetwork, schemaManager, dockerManager, - outputPath, + downloadManager, tempOutput ); } @@ -375,7 +224,7 @@ export async function validateTocContents( allowedAmount, schemaManager, dockerManager, - outputPath, + downloadManager, tempOutput ); } else { @@ -383,7 +232,7 @@ export async function validateTocContents( allowedAmount, schemaManager, dockerManager, - outputPath, + downloadManager, tempOutput ); } @@ -395,7 +244,7 @@ async function validateInNetworkFixedVersion( inNetwork: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - outputPath: string, + downloadManager: DownloadManager, tempOutput: string ) { const providerReferences: Set = new Set(); @@ -403,9 +252,9 @@ async function validateInNetworkFixedVersion( if (schemaPath != null) { for (const dataUrl of inNetwork) { try { - if (await checkDataUrl(dataUrl)) { + if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); - const dataPath = await downloadDataFile(dataUrl, temp.mkdirSync()); + const dataPath = await downloadManager.downloadDataFile(dataUrl); if (typeof dataPath === 'string') { // check if detected version matches the provided version // if there's no version property, that's ok @@ -434,7 +283,11 @@ async function validateInNetworkFixedVersion( ); } if (tempOutput.length > 0) { - appendResults(tempOutput, outputPath, `${dataUrl} - in-network${EOL}`); + appendResults( + tempOutput, + dockerManager.outputPath, + `${dataUrl} - in-network${EOL}` + ); } } } else { @@ -455,15 +308,15 @@ async function validateInNetworkDetectedVersion( inNetwork: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - outputPath: string, + downloadManager: DownloadManager, tempOutput: string ) { const providerReferences: Set = new Set(); for (const dataUrl of inNetwork) { try { - if (await checkDataUrl(dataUrl)) { + if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); - const dataPath = await downloadDataFile(dataUrl, temp.mkdirSync()); + const dataPath = await downloadManager.downloadDataFile(dataUrl); if (typeof dataPath === 'string') { const versionToUse = await schemaManager.determineVersion(dataPath); await schemaManager @@ -495,7 +348,11 @@ async function validateInNetworkDetectedVersion( ); } if (tempOutput.length > 0) { - appendResults(tempOutput, outputPath, `${dataUrl} - in-network${EOL}`); + appendResults( + tempOutput, + dockerManager.outputPath, + `${dataUrl} - in-network${EOL}` + ); } }); } @@ -511,16 +368,16 @@ async function validateAllowedAmountsFixedVersion( allowedAmount: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - outputPath: string, + downloadManager: DownloadManager, tempOutput: string ) { await schemaManager.useSchema('allowed-amounts').then(async schemaPath => { if (schemaPath != null) { for (const dataUrl of allowedAmount) { try { - if (await checkDataUrl(dataUrl)) { + if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); - const dataPath = await downloadDataFile(dataUrl, temp.mkdirSync()); + const dataPath = await downloadManager.downloadDataFile(dataUrl); if (typeof dataPath === 'string') { // check if detected version matches the provided version // if there's no version property, that's ok @@ -536,7 +393,11 @@ async function validateAllowedAmountsFixedVersion( .catch(() => {}); await dockerManager.runContainer(schemaPath, 'allowed-amounts', dataPath, tempOutput); if (tempOutput.length > 0) { - appendResults(tempOutput, outputPath, `${dataUrl} - allowed-amounts${EOL}`); + appendResults( + tempOutput, + dockerManager.outputPath, + `${dataUrl} - allowed-amounts${EOL}` + ); } } } else { @@ -556,14 +417,14 @@ async function validateAllowedAmountsDetectedVersion( allowedAmount: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - outputPath: string, + downloadManager: DownloadManager, tempOutput: string ) { for (const dataUrl of allowedAmount) { try { - if (await checkDataUrl(dataUrl)) { + if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); - const dataPath = await downloadDataFile(dataUrl, temp.mkdirSync()); + const dataPath = await downloadManager.downloadDataFile(dataUrl); if (typeof dataPath === 'string') { const versionToUse = await schemaManager.determineVersion(dataPath); await schemaManager @@ -587,7 +448,11 @@ async function validateAllowedAmountsDetectedVersion( }) .then(_containedResult => { if (tempOutput.length > 0) { - appendResults(tempOutput, outputPath, `${dataUrl} - allowed-amounts${EOL}`); + appendResults( + tempOutput, + dockerManager.outputPath, + `${dataUrl} - allowed-amounts${EOL}` + ); } }); } @@ -602,7 +467,7 @@ export async function assessReferencedProviders( providerReferences: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - outputPath: string + downloadManager: DownloadManager ) { if (providerReferences.length > 0) { const fileText = providerReferences.length === 1 ? 'this file' : 'these files'; @@ -610,15 +475,15 @@ export async function assessReferencedProviders( logger.info(`In-network file(s) refer to ${fileText}:`); logger.info('== Provider Reference =='); providerReferences.forEach(prf => logger.info(`* ${prf}`)); - const wantToValidateProviders = readlineSync.keyInYNStrict( - `Would you like to validate ${fileText}?` - ); + const wantToValidateProviders = + downloadManager.alwaysYes || + readlineSync.keyInYNStrict(`Would you like to validate ${fileText}?`); if (wantToValidateProviders) { await validateReferencedProviders( providerReferences, schemaManager, dockerManager, - outputPath + downloadManager ); } } @@ -629,11 +494,11 @@ export async function validateReferencedProviders( providerReferences: string[], schemaManager: SchemaManager, dockerManager: DockerManager, - outputPath: string + downloadManager: DownloadManager ) { temp.track(); let tempOutput = ''; - if (outputPath?.length > 0) { + if (dockerManager.outputPath?.length > 0) { tempOutput = path.join(temp.mkdirSync('providers'), 'contained-result'); } if (providerReferences.length > 0) { @@ -641,9 +506,9 @@ export async function validateReferencedProviders( if (schemaPath != null) { for (const dataUrl of providerReferences) { try { - if (await checkDataUrl(dataUrl)) { + if (await downloadManager.checkDataUrl(dataUrl)) { logger.info(`File: ${dataUrl}`); - const dataPath = await downloadDataFile(dataUrl, temp.mkdirSync()); + const dataPath = await downloadManager.downloadDataFile(dataUrl); if (typeof dataPath === 'string') { const containedResult = await dockerManager.runContainer( schemaPath, @@ -652,7 +517,11 @@ export async function validateReferencedProviders( tempOutput ); if (tempOutput.length > 0) { - appendResults(tempOutput, outputPath, `${dataUrl} - provider-reference${EOL}`); + appendResults( + tempOutput, + dockerManager.outputPath, + `${dataUrl} - provider-reference${EOL}` + ); } } } else { diff --git a/test/utils.test.ts b/test/DownloadManager.test.ts similarity index 65% rename from test/utils.test.ts rename to test/DownloadManager.test.ts index 2e05d36e..be152a96 100644 --- a/test/utils.test.ts +++ b/test/DownloadManager.test.ts @@ -4,12 +4,20 @@ import temp from 'temp'; import nock from 'nock'; import readlineSync from 'readline-sync'; import fs from 'fs-extra'; +import yauzl from 'yauzl'; +import { DownloadManager } from '../src/DownloadManager'; +import { ZipContents } from '../src/utils'; -import * as validatorUtils from '../src/utils'; +describe('DownloadManager', () => { + let downloadManager: DownloadManager; -describe('utils', () => { beforeAll(() => { temp.track(); + downloadManager = new DownloadManager(); + }); + + beforeEach(() => { + downloadManager.alwaysYes = false; }); describe('#checkDataUrl', () => { @@ -29,7 +37,7 @@ describe('utils', () => { it('should return true when the url is valid and the content length is less than one GB', async () => { nock('http://example.org').head('/data.json').reply(200, '', { 'content-length': '500' }); - const result = await validatorUtils.checkDataUrl('http://example.org/data.json'); + const result = await downloadManager.checkDataUrl('http://example.org/data.json'); expect(result).toBeTrue(); }); @@ -38,7 +46,16 @@ describe('utils', () => { .head('/data.json') .reply(200, '', { 'content-length': (Math.pow(1024, 3) * 2).toString() }); keyInYNStrictSpy.mockReturnValueOnce(true); - const result = await validatorUtils.checkDataUrl('http://example.org/data.json'); + const result = await downloadManager.checkDataUrl('http://example.org/data.json'); + expect(result).toBeTrue(); + }); + + it('should return true when the url is valid and and alwaysYes is true with a content length greater than one GB', async () => { + nock('http://example.org') + .head('/data.json') + .reply(200, '', { 'content-length': (Math.pow(1024, 3) * 2).toString() }); + downloadManager.alwaysYes = true; + const result = await downloadManager.checkDataUrl('http://example.org/data.json'); expect(result).toBeTrue(); }); @@ -47,27 +64,34 @@ describe('utils', () => { .head('/data.json') .reply(200, '', { 'content-length': (Math.pow(1024, 3) * 2).toString() }); keyInYNStrictSpy.mockReturnValueOnce(false); - const result = await validatorUtils.checkDataUrl('http://example.org/data.json'); + const result = await downloadManager.checkDataUrl('http://example.org/data.json'); expect(result).toBeFalse(); }); it('should return true when the url is valid and the user approves an unknown content length', async () => { nock('http://example.org').head('/data.json').reply(200); keyInYNStrictSpy.mockReturnValueOnce(true); - const result = await validatorUtils.checkDataUrl('http://example.org/data.json'); + const result = await downloadManager.checkDataUrl('http://example.org/data.json'); + expect(result).toBeTrue(); + }); + + it('should return true when the url is valid and alwaysYes is true with an unknown content length', async () => { + nock('http://example.org').head('/data.json').reply(200); + downloadManager.alwaysYes = true; + const result = await downloadManager.checkDataUrl('http://example.org/data.json'); expect(result).toBeTrue(); }); it('should return false when the url is valid and the user rejects an unknown content length', async () => { nock('http://example.org').head('/data.json').reply(200); keyInYNStrictSpy.mockReturnValueOnce(false); - const result = await validatorUtils.checkDataUrl('http://example.org/data.json'); + const result = await downloadManager.checkDataUrl('http://example.org/data.json'); expect(result).toBeFalse(); }); it('should return false when the url is not valid', async () => { nock('http://example.org').head('/data.json').reply(404); - const result = await validatorUtils.checkDataUrl('http://example.org/data.json'); + const result = await downloadManager.checkDataUrl('http://example.org/data.json'); expect(result).toBeFalse(); }); }); @@ -77,6 +101,21 @@ describe('utils', () => { nock.cleanAll(); }); + it('should write a file to the default folder', async () => { + const simpleData = fs.readJsonSync( + path.join(__dirname, 'fixtures', 'simpleData.json'), + 'utf-8' + ); + nock('http://example.org').get('/data.json').reply(200, simpleData); + const dataPath = (await downloadManager.downloadDataFile( + 'http://example.org/data.json' + )) as string; + expect(dataPath).toBeString(); + expect(fs.existsSync(dataPath)); + const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); + expect(downloadedData).toEqual(simpleData); + }); + it('should write a file to the specified folder', async () => { const simpleData = fs.readJsonSync( path.join(__dirname, 'fixtures', 'simpleData.json'), @@ -84,9 +123,13 @@ describe('utils', () => { ); nock('http://example.org').get('/data.json').reply(200, simpleData); const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/data.json', outputDir); - expect(fs.existsSync(path.join(outputDir, 'data.json'))); - const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); + const dataPath = (await downloadManager.downloadDataFile( + 'http://example.org/data.json', + outputDir + )) as string; + expect(dataPath).toBeString(); + expect(fs.existsSync(dataPath)); + const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); expect(downloadedData).toEqual(simpleData); }); @@ -99,10 +142,12 @@ describe('utils', () => { nock('http://example.org') .get('/data.gz') .reply(200, simpleGz, { 'content-type': 'application/gzip' }); - const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/data.gz', outputDir); - expect(fs.existsSync(path.join(outputDir, 'data.json'))); - const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); + const dataPath = (await downloadManager.downloadDataFile( + 'http://example.org/data.gz' + )) as string; + expect(dataPath).toBeString(); + expect(fs.existsSync(dataPath)); + const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); expect(downloadedData).toEqual(simpleData); }); @@ -115,10 +160,12 @@ describe('utils', () => { nock('http://example.org') .get('/data.gz') .reply(200, simpleGz, { 'content-type': 'application/octet-stream' }); - const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/data.gz', outputDir); - expect(fs.existsSync(path.join(outputDir, 'data.json'))); - const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); + const dataPath = (await downloadManager.downloadDataFile( + 'http://example.org/data.gz' + )) as string; + expect(dataPath).toBeString(); + expect(fs.existsSync(dataPath)); + const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); expect(downloadedData).toEqual(simpleData); }); @@ -132,13 +179,12 @@ describe('utils', () => { .get('/data.gz') .query(true) .reply(200, simpleGz, { 'content-type': 'application/octet-stream' }); - const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile( - 'http://example.org/data.gz?Expires=123456&mode=true', - outputDir - ); - expect(fs.existsSync(path.join(outputDir, 'data.json'))); - const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); + const dataPath = (await downloadManager.downloadDataFile( + 'http://example.org/data.gz?Expires=123456&mode=true' + )) as string; + expect(dataPath).toBeString(); + expect(fs.existsSync(dataPath)); + const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); expect(downloadedData).toEqual(simpleData); }); @@ -155,7 +201,7 @@ describe('utils', () => { .get('/data.gz') .reply(200, simpleGz, { 'content-type': 'application/octet-stream' }); const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/some-data', outputDir); + await downloadManager.downloadDataFile('http://example.org/some-data', outputDir); expect(fs.existsSync(path.join(outputDir, 'data.json'))); const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); expect(downloadedData).toEqual(simpleData); @@ -175,7 +221,7 @@ describe('utils', () => { .query(true) .reply(200, simpleGz, { 'content-type': 'application/octet-stream' }); const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/some-data', outputDir); + await downloadManager.downloadDataFile('http://example.org/some-data', outputDir); expect(fs.existsSync(path.join(outputDir, 'data.json'))); const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); expect(downloadedData).toEqual(simpleData); @@ -190,10 +236,12 @@ describe('utils', () => { nock('http://example.org') .get('/data.zip') .reply(200, simpleZip, { 'content-type': 'application/zip' }); - const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/data.zip', outputDir); - expect(fs.existsSync(path.join(outputDir, 'data.json'))); - const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); + const dataPath = (await downloadManager.downloadDataFile( + 'http://example.org/data.zip' + )) as string; + expect(dataPath).toBeString(); + expect(fs.existsSync(dataPath)); + const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); expect(downloadedData).toEqual(simpleData); }); @@ -206,10 +254,12 @@ describe('utils', () => { nock('http://example.org') .get('/data.zip') .reply(200, simpleZip, { 'content-type': 'application/octet-stream' }); - const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/data.zip', outputDir); - expect(fs.existsSync(path.join(outputDir, 'data.json'))); - const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); + const dataPath = (await downloadManager.downloadDataFile( + 'http://example.org/data.zip' + )) as string; + expect(dataPath).toBeString(); + expect(fs.existsSync(dataPath)); + const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); expect(downloadedData).toEqual(simpleData); }); @@ -223,16 +273,30 @@ describe('utils', () => { .get('/data.zip') .query(true) .reply(200, simpleZip, { 'content-type': 'application/octet-stream' }); - const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile( - 'http://example.org/data.zip?mode=on&rate=7', - outputDir - ); - expect(fs.existsSync(path.join(outputDir, 'data.json'))); - const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); + const dataPath = (await downloadManager.downloadDataFile( + 'http://example.org/data.zip?mode=on&rate=7' + )) as string; + expect(dataPath).toBeString(); + expect(fs.existsSync(dataPath)); + const downloadedData = fs.readJsonSync(dataPath, 'utf-8'); expect(downloadedData).toEqual(simpleData); }); + it('should return information about the zip contents when the zip has more than one json file', async () => { + const multiZip = fs.readFileSync(path.join(__dirname, 'fixtures', 'multiZip.zip')); + nock('http://example.org') + .get('/multi.zip') + .query(true) + .reply(200, multiZip, { 'content-type': 'application/zip' }); + const zipInfo = (await downloadManager.downloadDataFile( + 'http://example.org/multi.zip?mode=more' + )) as ZipContents; + expect(zipInfo).toBeObject(); + expect(zipInfo.zipFile).toBeInstanceOf(yauzl.ZipFile); + expect(zipInfo.jsonEntries).toHaveLength(2); + expect(zipInfo.jsonEntries[0].fileName).toBe('moreData.json'); + expect(zipInfo.jsonEntries[1].fileName).toBe('simpleData.json'); + }); it('should write a json file within a zip when the response has content type application/octet-stream and the url after redirection ends with .zip', async () => { const simpleData = fs.readJsonSync( path.join(__dirname, 'fixtures', 'simpleData.json'), @@ -246,7 +310,7 @@ describe('utils', () => { .get('/data.zip') .reply(200, simpleZip, { 'content-type': 'application/octet-stream' }); const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/data-please', outputDir); + await downloadManager.downloadDataFile('http://example.org/data-please', outputDir); expect(fs.existsSync(path.join(outputDir, 'data.json'))); const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); expect(downloadedData).toEqual(simpleData); @@ -266,7 +330,7 @@ describe('utils', () => { .query(true) .reply(200, simpleZip, { 'content-type': 'application/octet-stream' }); const outputDir = temp.mkdirSync(); - await validatorUtils.downloadDataFile('http://example.org/data-please', outputDir); + await downloadManager.downloadDataFile('http://example.org/data-please', outputDir); expect(fs.existsSync(path.join(outputDir, 'data.json'))); const downloadedData = fs.readJsonSync(path.join(outputDir, 'data.json'), 'utf-8'); expect(downloadedData).toEqual(simpleData); @@ -279,7 +343,7 @@ describe('utils', () => { .reply(200, wrongZip, { 'content-type': 'application/zip' }); const outputDir = temp.mkdirSync(); await expect( - validatorUtils.downloadDataFile('http://example.org/data.zip', outputDir) + downloadManager.downloadDataFile('http://example.org/data.zip', outputDir) ).toReject(); }); @@ -287,7 +351,7 @@ describe('utils', () => { nock('http://example.org').get('/data.json').reply(500); const outputDir = temp.mkdirSync(); await expect( - validatorUtils.downloadDataFile('http://example.org/data.json', outputDir) + downloadManager.downloadDataFile('http://example.org/data.json', outputDir) ).toReject(); }); }); diff --git a/test/commands.test.ts b/test/commands.test.ts index d4e7fab8..e4eba0cb 100644 --- a/test/commands.test.ts +++ b/test/commands.test.ts @@ -1,32 +1,52 @@ import 'jest-extended'; import path from 'path'; -import * as validatorUtils from '../src/utils'; import { validate, validateFromUrl } from '../src/commands'; import { SchemaManager } from '../src/SchemaManager'; import { DockerManager } from '../src/DockerManager'; +import { DownloadManager } from '../src/DownloadManager'; describe('commands', () => { - describe('#validate', () => { - let useVersionSpy: jest.SpyInstance; - let useSchemaSpy: jest.SpyInstance; - let runContainerSpy: jest.SpyInstance; + let checkDataUrlSpy: jest.SpyInstance; + let ensureRepoSpy: jest.SpyInstance; + let useVersionSpy: jest.SpyInstance; + let useSchemaSpy: jest.SpyInstance; + let downloadDataSpy: jest.SpyInstance; + let runContainerSpy: jest.SpyInstance; + + beforeAll(() => { + checkDataUrlSpy = jest.spyOn(DownloadManager.prototype, 'checkDataUrl'); + ensureRepoSpy = jest + .spyOn(SchemaManager.prototype, 'ensureRepo') + .mockResolvedValue({ stdout: '', stderr: '' }); + useVersionSpy = jest.spyOn(SchemaManager.prototype, 'useVersion').mockResolvedValue(true); + useSchemaSpy = jest.spyOn(SchemaManager.prototype, 'useSchema').mockResolvedValue('schemaPath'); + downloadDataSpy = jest + .spyOn(DownloadManager.prototype, 'downloadDataFile') + .mockResolvedValue('data.json'); + runContainerSpy = jest + .spyOn(DockerManager.prototype, 'runContainer') + .mockResolvedValue({ pass: false }); + }); - beforeAll(() => { - useVersionSpy = jest.spyOn(SchemaManager.prototype, 'useVersion').mockResolvedValue(true); - useSchemaSpy = jest - .spyOn(SchemaManager.prototype, 'useSchema') - .mockResolvedValue('schemaPath'); - runContainerSpy = jest - .spyOn(DockerManager.prototype, 'runContainer') - .mockResolvedValue({ pass: false }); - }); + beforeEach(() => { + checkDataUrlSpy.mockClear(); + ensureRepoSpy.mockClear(); + useVersionSpy.mockClear(); + useSchemaSpy.mockClear(); + downloadDataSpy.mockClear(); + runContainerSpy.mockClear(); + }); - beforeEach(() => { - useVersionSpy.mockClear(); - useSchemaSpy.mockClear(); - runContainerSpy.mockClear(); - }); + afterAll(() => { + checkDataUrlSpy.mockRestore(); + ensureRepoSpy.mockRestore(); + useVersionSpy.mockRestore(); + useSchemaSpy.mockRestore(); + downloadDataSpy.mockRestore(); + runContainerSpy.mockRestore(); + }); + describe('#validate', () => { it('should continue processing when the data file exists', async () => { await validate(path.join(__dirname, '..', 'test-files', 'allowed-amounts.json'), { target: null @@ -51,40 +71,9 @@ describe('commands', () => { await validate(path.join(__dirname, '..', 'test-files', 'not-real.json'), { target: null }); expect(runContainerSpy).toHaveBeenCalledTimes(0); }); - - afterAll(() => { - useVersionSpy.mockRestore(); - useSchemaSpy.mockRestore(); - runContainerSpy.mockRestore(); - }); }); describe('#validateFromUrl', () => { - let checkDataUrlSpy: jest.SpyInstance; - let useSchemaSpy: jest.SpyInstance; - let downloadDataSpy: jest.SpyInstance; - let runContainerSpy: jest.SpyInstance; - - beforeAll(() => { - checkDataUrlSpy = jest.spyOn(validatorUtils, 'checkDataUrl'); - useSchemaSpy = jest - .spyOn(SchemaManager.prototype, 'useSchema') - .mockResolvedValue('schemaPath'); - downloadDataSpy = jest - .spyOn(validatorUtils, 'downloadDataFile') - .mockResolvedValue('data.json'); - runContainerSpy = jest - .spyOn(DockerManager.prototype, 'runContainer') - .mockResolvedValue({ pass: false }); - }); - - beforeEach(() => { - checkDataUrlSpy.mockClear(); - useSchemaSpy.mockClear(); - downloadDataSpy.mockClear(); - runContainerSpy.mockClear(); - }); - it('should continue processing when the data url is valid and content length is less than or equal to the size limit', async () => { checkDataUrlSpy.mockResolvedValueOnce(true); await validateFromUrl('http://example.org/data.json', { schemaVersion: 'v1.0.0' }); @@ -106,16 +95,12 @@ describe('commands', () => { schemaVersion: 'v1.0.0' }); expect(downloadDataSpy).toHaveBeenCalledTimes(1); - expect(downloadDataSpy).toHaveBeenCalledWith( - 'http://example.org/data.json', - expect.any(String) - ); + expect(downloadDataSpy).toHaveBeenCalledWith('http://example.org/data.json'); expect(runContainerSpy).toHaveBeenCalledTimes(1); expect(runContainerSpy).toHaveBeenCalledWith( 'schemapath.json', 'in-network-rates', - 'data.json', - undefined + 'data.json' ); }); @@ -125,12 +110,5 @@ describe('commands', () => { await validateFromUrl('http://example.org/data.json', { schemaVersion: 'v1.0.0' }); expect(downloadDataSpy).toHaveBeenCalledTimes(0); }); - - afterAll(() => { - checkDataUrlSpy.mockRestore(); - useSchemaSpy.mockRestore(); - downloadDataSpy.mockRestore(); - runContainerSpy.mockRestore(); - }); }); }); diff --git a/test/fixtures/moreData.json b/test/fixtures/moreData.json new file mode 100644 index 00000000..76e3bf5f --- /dev/null +++ b/test/fixtures/moreData.json @@ -0,0 +1,4 @@ +{ + "data": "This is a simple JSON file.", + "cookie": true +} \ No newline at end of file diff --git a/test/fixtures/multiZip.zip b/test/fixtures/multiZip.zip new file mode 100644 index 00000000..d72bb871 Binary files /dev/null and b/test/fixtures/multiZip.zip differ