diff --git a/packages/cubejs-backend-cloud/src/cloud.ts b/packages/cubejs-backend-cloud/src/cloud.ts index 93d2dfd31073b..2c14e3cff8d3a 100644 --- a/packages/cubejs-backend-cloud/src/cloud.ts +++ b/packages/cubejs-backend-cloud/src/cloud.ts @@ -113,22 +113,18 @@ export class CubeCloudClient { const formData = new FormData(); formData.append('transaction', JSON.stringify(transaction)); formData.append('fileName', fileName); - formData.append('file', { - value: data, - options: { - filename: path.basename(fileName), - contentType: 'application/octet-stream' - } + formData.append('file', data, { + filename: path.basename(fileName), + contentType: 'application/octet-stream' }); // Get the form data buffer and headers - const formDataBuffer = formData.getBuffer(); const formDataHeaders = formData.getHeaders(); return this.request({ url: (deploymentId: string) => `build/deploy/${deploymentId}/upload-file${this.extendRequestByLivePreview()}`, method: 'POST', - body: formDataBuffer, + body: formData, headers: { ...formDataHeaders, }, diff --git a/packages/cubejs-backend-cloud/src/deploy.ts b/packages/cubejs-backend-cloud/src/deploy.ts index 12f43323922db..9abcfc0fa1049 100644 --- a/packages/cubejs-backend-cloud/src/deploy.ts +++ b/packages/cubejs-backend-cloud/src/deploy.ts @@ -1,6 +1,7 @@ import crypto from 'crypto'; import path from 'path'; import fs from 'fs-extra'; +import { DotenvParseOutput } from '@cubejs-backend/dotenv'; import { CubeCloudClient } from './cloud'; type DeployDirectoryOptions = { @@ -64,10 +65,10 @@ export class DeployDirectory { } type DeployHooks = { - onStart?: Function, - onUpdate?: Function, - onUpload?: Function, - onFinally?: Function + onStart?: (deploymentName: string, files: string[]) => void, + onUpdate?: (i: number, { file }: { file: string}) => void, + onUpload?: (files: string[], file: string) => void, + onFinally?: () => void }; export interface DeployResponse { @@ -78,6 +79,7 @@ export interface DeployResponse { export class DeployController { public constructor( protected readonly cubeCloudClient: CubeCloudClient, + protected readonly envVariables: DotenvParseOutput = {}, protected readonly hooks: DeployHooks = {} ) { } @@ -90,6 +92,10 @@ export class DeployController { const upstreamHashes = await this.cubeCloudClient.getUpstreamHashes(); const { transaction, deploymentName } = await this.cubeCloudClient.startUpload(); + if (Object.keys(this.envVariables).length) { + await this.cubeCloudClient.setEnvVars({ envVariables: this.envVariables }); + } + const files = Object.keys(fileHashes); const fileHashesPosix: Record = {}; if (this.hooks.onStart) this.hooks.onStart(deploymentName, files); diff --git a/packages/cubejs-cli/package.json b/packages/cubejs-cli/package.json index b8cfc9a09d857..c1d3fe25d2f8d 100644 --- a/packages/cubejs-cli/package.json +++ b/packages/cubejs-cli/package.json @@ -31,6 +31,7 @@ ], "dependencies": { "@cubejs-backend/dotenv": "^9.0.2", + "@cubejs-backend/cloud": "1.1.13", "@cubejs-backend/schema-compiler": "1.1.15", "@cubejs-backend/shared": "1.1.12", "chalk": "^2.4.2", diff --git a/packages/cubejs-cli/src/command/auth.ts b/packages/cubejs-cli/src/command/auth.ts index fb771a0713da9..45721fc895316 100644 --- a/packages/cubejs-cli/src/command/auth.ts +++ b/packages/cubejs-cli/src/command/auth.ts @@ -1,7 +1,7 @@ import type { CommanderStatic } from 'commander'; +import { Config } from '@cubejs-backend/cloud'; import { displayError, event } from '../utils'; -import { Config } from '../config'; const authenticate = async (currentToken: string) => { const config = new Config(); diff --git a/packages/cubejs-cli/src/command/deploy.ts b/packages/cubejs-cli/src/command/deploy.ts index 68ea8d903be5d..e391c8559a173 100644 --- a/packages/cubejs-cli/src/command/deploy.ts +++ b/packages/cubejs-cli/src/command/deploy.ts @@ -1,23 +1,11 @@ -import FormData from 'form-data'; import fs from 'fs-extra'; import path from 'path'; import cliProgress from 'cli-progress'; import { CommanderStatic } from 'commander'; +import { CubeCloudClient, DeployController } from '@cubejs-backend/cloud'; +import { ConfigCli } from '../config'; -import { DeployDirectory } from '../deploy'; import { logStage, displayError, event } from '../utils'; -import { Config } from '../config'; - -interface Hashes { - [key: string]: { - hash: string; - }; -} - -interface CloudReqResult { - transaction: string; - deploymentName: string; -} const deploy = async ({ directory, auth, uploadEnv, token }: any) => { if (!(await fs.pathExists(path.join(process.cwd(), 'node_modules', '@cubejs-backend/server-core')))) { @@ -26,18 +14,13 @@ const deploy = async ({ directory, auth, uploadEnv, token }: any) => { ); } + const config = new ConfigCli(); if (token) { - const config = new Config(); await config.addAuthToken(token); - - await event({ - event: 'Cube Cloud CLI Authenticate' - }); - + await event({ event: 'Cube Cloud CLI Authenticate' }); console.log('Token successfully added!'); } - const config = new Config(); const bar = new cliProgress.SingleBar({ format: '- Uploading files | {bar} | {percentage}% || {value} / {total} | {file}', barCompleteChar: '\u2588', @@ -45,86 +28,28 @@ const deploy = async ({ directory, auth, uploadEnv, token }: any) => { hideCursor: true }); - const deployDir = new DeployDirectory({ directory }); - const fileHashes: any = await deployDir.fileHashes(); - - const upstreamHashes: Hashes = await config.cloudReq({ - url: (deploymentId: string) => `build/deploy/${deploymentId}/files`, - method: 'GET', - auth - }); - - const { transaction, deploymentName }: CloudReqResult = await config.cloudReq({ - url: (deploymentId: string) => `build/deploy/${deploymentId}/start-upload`, - method: 'POST', - auth - }); - - if (uploadEnv) { - const envVariables = await config.envFile(`${directory}/.env`); - await config.cloudReq({ - url: (deploymentId) => `build/deploy/${deploymentId}/set-env`, - method: 'POST', - body: JSON.stringify({ - envVariables: JSON.stringify(envVariables), - }), - headers: { - 'Content-type': 'application/json' - }, - auth - }); - } - - await logStage(`Deploying ${deploymentName}...`, 'Cube Cloud CLI Deploy'); - - const files = Object.keys(fileHashes); - const fileHashesPosix = {}; - - bar.start(files.length, 0, { - file: '' - }); - - try { - for (let i = 0; i < files.length; i++) { - const file = files[i]; + const envVariables = uploadEnv ? await config.envFile(`${directory}/.env`) : {}; + + const cubeCloudClient = new CubeCloudClient(auth || (await config.deployAuthForCurrentDir())); + const deployController = new DeployController(cubeCloudClient, envVariables, { + onStart: async (deploymentName, files) => { + await logStage(`Deploying ${deploymentName}...`, 'Cube Cloud CLI Deploy'); + bar.start(files.length, 0, { + file: '' + }); + }, + onUpdate: (i, { file }) => { bar.update(i, { file }); - - const filePosix = file.split(path.sep).join(path.posix.sep); - fileHashesPosix[filePosix] = fileHashes[file]; - - if (!upstreamHashes[filePosix] || upstreamHashes[filePosix].hash !== fileHashes[file].hash) { - const formData = new FormData(); - formData.append('transaction', JSON.stringify(transaction)); - formData.append('fileName', filePosix); - formData.append('file', fs.createReadStream(path.join(directory, file)), { - filename: path.basename(file), - contentType: 'application/octet-stream' - }); - - await config.cloudReq({ - url: (deploymentId: string) => `build/deploy/${deploymentId}/upload-file`, - method: 'POST', - body: formData, - auth, - headers: formData.getHeaders() - }); - } + }, + onUpload: (files) => { + bar.update(files.length, { file: 'Post processing...' }); + }, + onFinally: () => { + bar.stop(); } - bar.update(files.length, { file: 'Post processing...' }); - await config.cloudReq({ - url: (deploymentId: string) => `build/deploy/${deploymentId}/finish-upload`, - method: 'POST', - body: JSON.stringify({ - transaction, - files: fileHashesPosix - }), - headers: { 'Content-type': 'application/json' }, - auth - }); - } finally { - bar.stop(); - } + }); + await deployController.deploy(directory); await logStage('Done 🎉', 'Cube Cloud CLI Deploy Success'); }; diff --git a/packages/cubejs-cli/src/config.ts b/packages/cubejs-cli/src/config.ts index 0a889299ddf97..d389d3666023a 100644 --- a/packages/cubejs-cli/src/config.ts +++ b/packages/cubejs-cli/src/config.ts @@ -1,89 +1,7 @@ import inquirer from 'inquirer'; -import fs from 'fs-extra'; -import fetch, { RequestInit } from 'node-fetch'; -import jwt from 'jsonwebtoken'; -import path from 'path'; -import os from 'os'; -import dotenv from '@cubejs-backend/dotenv'; -import { isFilePath } from '@cubejs-backend/shared'; -import { displayWarning } from './utils'; - -type ConfigurationFull = { - auth: { - [organizationUrl: string]: { - auth: string, - } - } -}; - -type Configuration = Partial; - -export class Config { - protected async loadConfig(): Promise { - const { configFile } = this.configFile(); - - if (await fs.pathExists(configFile)) { - return fs.readJson(configFile); - } - - return {}; - } - - protected async writeConfig(config) { - const { cubeCloudConfigPath, configFile } = this.configFile(); - await fs.mkdirp(cubeCloudConfigPath); - await fs.writeJson(configFile, config); - } - - protected configFile() { - const cubeCloudConfigPath = this.cubeCloudConfigPath(); - const configFile = path.join(cubeCloudConfigPath, 'config.json'); - - return { cubeCloudConfigPath, configFile }; - } - - public async envFile(envFile: string) { - if (await fs.pathExists(envFile)) { - const env = dotenv.config({ path: envFile, multiline: 'line-breaks' }).parsed; - if (env) { - if ('CUBEJS_DEV_MODE' in env) { - delete env.CUBEJS_DEV_MODE; - } - - const resolvePossibleFiles = [ - 'CUBEJS_DB_SSL_CA', - 'CUBEJS_DB_SSL_CERT', - 'CUBEJS_DB_SSL_KEY', - ]; - - // eslint-disable-next-line no-restricted-syntax - for (const [key, value] of Object.entries(env)) { - if (resolvePossibleFiles.includes(key) && isFilePath(value)) { - if (fs.existsSync(value)) { - env[key] = fs.readFileSync(value, 'ascii'); - } else { - displayWarning(`Unable to resolve file "${value}" from ${key}`); - - env[key] = ''; - } - } - } - - return env; - } - } - - return {}; - } - - protected cubeEnvConfigPath() { - return path.join(os.homedir(), '.env'); - } - - protected cubeCloudConfigPath() { - return path.join(os.homedir(), '.cubecloud'); - } +import { Config } from '@cubejs-backend/cloud'; +export class ConfigCli extends Config { public async deployAuth(url?: string) { const config = await this.loadConfig(); @@ -103,39 +21,7 @@ export class Config { return (await this.addAuthToken(auth.auth, config)).auth; } - public async addAuthToken(authToken: string, config?: Configuration): Promise { - if (!config) { - config = await this.loadConfig(); - } - - const payload = jwt.decode(authToken); - if (payload && typeof payload === 'object' && payload.url) { - config.auth = config.auth || {}; - config.auth[payload.url] = { - auth: authToken - }; - - if (payload.deploymentId) { - const dotCubeCloud = await this.loadDotCubeCloud(); - dotCubeCloud.url = payload.url; - dotCubeCloud.deploymentId = payload.deploymentId; - await this.writeDotCubeCloud(dotCubeCloud); - } - - await this.writeConfig(config); - return config; - } - - const answer = await this.cloudTokenReq(authToken); - if (answer) { - return this.addAuthToken(answer, config); - } - - // eslint-disable-next-line no-throw-literal - throw 'Malformed Cube Cloud token'; - } - - protected async deployAuthForCurrentDir() { + public async deployAuthForCurrentDir() { const dotCubeCloud = await this.loadDotCubeCloud(); if (dotCubeCloud.url && dotCubeCloud.deploymentId) { const deployAuth = await this.deployAuth(dotCubeCloud.url); @@ -163,11 +49,7 @@ export class Config { } const authToken = auth[url]; - const deployments = await this.cloudReq({ - url: () => 'build/deploy/deployments', - method: 'GET', - auth: { ...authToken, url } - }); + const deployments = await this.cubeCloudClient.getDeploymentsList({ auth: { ...authToken, url } }); if (!Array.isArray(deployments)) { throw new Error(JSON.stringify(deployments)); @@ -200,71 +82,4 @@ export class Config { deploymentId }; } - - protected dotCubeCloudFile() { - return '.cubecloud'; - } - - protected async loadDotCubeCloud() { - if (await fs.pathExists(this.dotCubeCloudFile())) { - return fs.readJson(this.dotCubeCloudFile()); - } - - return {}; - } - - protected async writeDotCubeCloud(config) { - await fs.writeJson(this.dotCubeCloudFile(), config); - } - - public async cloudReq(options: { - url: (deploymentId: string) => string, - auth: { auth: string, deploymentId?: string, url?: string }, - } & RequestInit): Promise { - const { url, auth, ...restOptions } = options; - - const authorization = auth || await this.deployAuthForCurrentDir(); - if (!authorization) { - throw new Error('Auth isn\'t set'); - } - - // Ensure headers object exists in restOptions - restOptions.headers = restOptions.headers || {}; - // Add authorization to headers - (restOptions.headers as any).authorization = authorization.auth; - - const response = await fetch( - `${authorization.url}/${url(authorization.deploymentId || '')}`, - restOptions, - ); - - if (!response.ok) { - throw new Error(`HTTP error! status: ${response.status}`); - } - - return await response.json() as Promise; - } - - protected async cloudTokenReq(authToken: string) { - const res = await fetch( - `${process.env.CUBE_CLOUD_HOST || 'https://cubecloud.dev'}/v1/token`, - { - method: 'POST', - headers: { 'Content-type': 'application/json' }, - body: JSON.stringify({ token: authToken }) - } - ); - - if (!res.ok) { - throw new Error(`HTTP error! status: ${res.status}`); - } - - const response = await res.json() as any; - - if (!response.jwt) { - throw new Error('JWT token is not present in the response'); - } - - return response.jwt; - } } diff --git a/packages/cubejs-cli/src/deploy.ts b/packages/cubejs-cli/src/deploy.ts deleted file mode 100644 index 1b6158f83e12b..0000000000000 --- a/packages/cubejs-cli/src/deploy.ts +++ /dev/null @@ -1,63 +0,0 @@ -import crypto from 'crypto'; -import fs from 'fs-extra'; -import path from 'path'; - -type DeployDirectoryOptions = { - directory: string, -}; - -export class DeployDirectory { - public constructor( - protected readonly options: DeployDirectoryOptions - ) { } - - public async fileHashes(directory: string = this.options.directory) { - let result = {}; - - const files = await fs.readdir(directory); - // eslint-disable-next-line no-restricted-syntax - for (const file of files) { - const filePath = path.resolve(directory, file); - if (!this.filter(filePath)) { - // eslint-disable-next-line no-continue - continue; - } - const stat = await fs.stat(filePath); - if (stat.isDirectory()) { - result = { ...result, ...await this.fileHashes(filePath) }; - } else { - result[path.relative(this.options.directory, filePath)] = { - hash: await this.fileHash(filePath) - }; - } - } - return result; - } - - protected filter(file: string) { - const baseName = path.basename(file); - - // whitelist - if (['.gitignore'].includes(baseName)) { - return true; - } - - // blacklist - if (['dashboard-app', 'node_modules'].includes(baseName)) { - return false; - } - - return baseName.charAt(0) !== '.'; - } - - protected fileHash(file: string) { - return new Promise((resolve, reject) => { - const hash = crypto.createHash('sha1'); - const stream = fs.createReadStream(file); - - stream.on('error', err => reject(err)); - stream.on('data', chunk => hash.update(chunk)); - stream.on('end', () => resolve(hash.digest('hex'))); - }); - } -}