From 2880dd9b9f8bf45004bd999f3f2005c6ad0b1490 Mon Sep 17 00:00:00 2001 From: joelchu Date: Sun, 6 Oct 2019 12:07:32 +0800 Subject: [PATCH 1/4] Changing the top level input to make more sense --- .../src/generator/get-resolver.js | 10 ++--- .../generator/hamsters/preprocess-resolver.js | 14 +++---- packages/contract-cli/src/generator/index.js | 6 +-- .../src/generator/process-file.js | 12 +++--- .../src/generator/read-files-out-contract.js | 18 ++++---- .../contract-cli/src/generator/split-task.js | 13 +++--- packages/contract-cli/src/utils.js | 2 - .../contract-cli/tests/split-task.test.js | 42 ++++++++++++++----- 8 files changed, 69 insertions(+), 48 deletions(-) diff --git a/packages/contract-cli/src/generator/get-resolver.js b/packages/contract-cli/src/generator/get-resolver.js index 3a67ed1f..78e52283 100644 --- a/packages/contract-cli/src/generator/get-resolver.js +++ b/packages/contract-cli/src/generator/get-resolver.js @@ -122,15 +122,16 @@ const checkResolver = (indexFile, inDir, fileType, config) => (baseFile) => { /** * filter to get which is resolver or not - * @param {string} inDir the base path - * @param {string} fileType ext * @param {object} config options to use + * @param {string} fileType ext * @return {function} work out if it's or not */ -function getResolver(inDir, fileType, config) { +function getResolver(config, fileType) { const indexFile = [INDEX_KEY, fileType].join('.') // return a function here - return checkResolver(indexFile, inDir, fileType, config) + const fn = checkResolver(indexFile, config.resolverDir, fileType, config) + debug(fn.toString()) + return fn; } /** @@ -149,7 +150,6 @@ function processResolverToContract(type, name, file, public, namespace, sourceTy const source = fsx.readFileSync(file, 'utf8').toString() // parsing the file const result = astParser(source, { sourceType }) - // logToFile(config.logDirectory, [type, name].join('-'), result) // get params @BUG the error happens here? const baseParams = result.body.filter(isExpression).map(extractParams).filter(p => p) diff --git a/packages/contract-cli/src/generator/hamsters/preprocess-resolver.js b/packages/contract-cli/src/generator/hamsters/preprocess-resolver.js index 3cb30fb6..8ecb8f87 100644 --- a/packages/contract-cli/src/generator/hamsters/preprocess-resolver.js +++ b/packages/contract-cli/src/generator/hamsters/preprocess-resolver.js @@ -7,11 +7,11 @@ const { getResolver } = require('../get-resolver') * @param {string} payload.file the argument pass to the processor * @return {void} just use process.send back the idx, processor and result */ -process.on('message', ({file, inDir, fileType, config}) => { - const preprocessor = getResolver(inDir, fileType, config) - Reflect.apply(preprocessor, null, [file]) - .then(result => { - // send the result straight back - process.send(result) - }) +process.on('message', ({file, config, fileType}) => { + const fn = getResolver(config, fileType) + fn(file) + .then(result => { + // send the result straight back + process.send(result) + }) }) diff --git a/packages/contract-cli/src/generator/index.js b/packages/contract-cli/src/generator/index.js index 3bc41c1f..6d6afb0d 100644 --- a/packages/contract-cli/src/generator/index.js +++ b/packages/contract-cli/src/generator/index.js @@ -10,7 +10,7 @@ const { merge } = require('lodash') const colors = require('colors/safe') const debug = require('debug')('jsonql-contract:generator') -const { EXT, PUBLIC_CONTRACT_FILE_NAME } = require('jsonql-constants') +const { PUBLIC_CONTRACT_FILE_NAME } = require('jsonql-constants') const { publicContractGenerator } = require('../public-contract') const { readFilesOutContract } = require('./read-files-out-contract') @@ -59,8 +59,8 @@ const callPublicGenerator = (config, contract) => ( * @return {mixed} depends on the returnAs parameter */ const generateNewContract = (config) => { - const { resolverDir, contractDir } = config; - return readFilesOutContract(resolverDir, config, config.ext || EXT) + // const { resolverDir, contractDir } = config; + return readFilesOutContract(config) .then(([sourceType, contract]) => { if (config.public === true) { // we can get the sourceType from the contract diff --git a/packages/contract-cli/src/generator/process-file.js b/packages/contract-cli/src/generator/process-file.js index 6f5335b1..b2c87bf2 100644 --- a/packages/contract-cli/src/generator/process-file.js +++ b/packages/contract-cli/src/generator/process-file.js @@ -44,7 +44,7 @@ function processFilesAction(inDir, fileType, files, config) { return callPreSplitTask(inDir, fileType, files, config) .then(result => { if (result === false) { - const preprocessor = getResolver(inDir, fileType, config) + const preprocessor = getResolver(config, fileType) return files.map( preprocessor ) .filter( obj => obj.ok ) .reduce( (n, m) => [...n, m], []) // do we need this? @@ -55,13 +55,13 @@ function processFilesAction(inDir, fileType, files, config) { /** * This is the step one to parallel process the resolver file + * @param {array} files to process * @param {string} inDir where the resolverDir is * @param {string} fileType extension to expect - * @param {array} _files to process * @param {object} config options * @return {array} of preprocessed filtered resolver */ -function preprocessResolverFile(inDir, fileType, files, config) { +function preprocessResolverFile(files, inDir, fileType, config) { return processFilesAction(inDir, fileType, files, config) .then(objs => [ getSourceType(objs), @@ -73,12 +73,12 @@ function preprocessResolverFile(inDir, fileType, files, config) { * process the files * @param {string} inDir base directory * @param {string} fileType the ext - * @param {array} _files list of files + * @param {array} files list of files * @param {object} config to control the inner working * @return {object} promise that resolve all the files key query / mutation */ -function processResolverFile(sourceType, _files, config) { - return callPostSplitTask(sourceType, _files, config) +function processResolverFile(sourceType, files, config) { + return callPostSplitTask(sourceType, files, config) .then(result => ( result || _files .map(({ type, name, file, public, namespace }) => ( diff --git a/packages/contract-cli/src/generator/read-files-out-contract.js b/packages/contract-cli/src/generator/read-files-out-contract.js index e9bc1281..38638e60 100644 --- a/packages/contract-cli/src/generator/read-files-out-contract.js +++ b/packages/contract-cli/src/generator/read-files-out-contract.js @@ -15,12 +15,12 @@ const debug = getDebug('read-files-out-contract') /** * just return the list of files for process - * @param {string} inDir the path to the resolver directory + * @param {string} resolverDir the path to the resolver directory * @param {string} [fileType=EXT] the file extension * @return {promise} resolve the files array on success or reject with error */ -function getResolverFiles(inDir, fileType = EXT) { - const pat = join(inDir, '**', ['*', fileType].join('.')) +function getResolverFiles(resolverDir, fileType) { + const pat = join(resolverDir, '**', ['*', fileType].join('.')) return new Promise((resolver, rejecter) => { glob(pat, (err, files) => { if (err) { @@ -34,16 +34,16 @@ function getResolverFiles(inDir, fileType = EXT) { /** * get list of files and put them in query / mutation - * @param {string} inDir diretory * @param {object} config pass config to the underlying method - * @param {mixed} fileType we might do a ts in the future * @return {object} query / mutation */ -function readFilesOutContract(inDir, config, fileType) { +function readFilesOutContract(config) { + const { resolverDir } = config; + let fileType = config.ext || EXT; let timestart = Date.now() - return getResolverFiles(inDir, fileType) + return getResolverFiles(resolverDir, fileType) .then(_files => ( - preprocessResolverFile(inDir, fileType, _files, config) + preprocessResolverFile(_files, resolverDir, fileType, config) )) .then(result => { const [sourceType, files] = result; @@ -58,6 +58,6 @@ function readFilesOutContract(inDir, config, fileType) { } module.exports = { - getResolverFiles, // export for testing + getResolverFiles, readFilesOutContract } diff --git a/packages/contract-cli/src/generator/split-task.js b/packages/contract-cli/src/generator/split-task.js index 0f0d258e..59524c23 100644 --- a/packages/contract-cli/src/generator/split-task.js +++ b/packages/contract-cli/src/generator/split-task.js @@ -25,6 +25,9 @@ function getSlots() { */ function executeTask(taskPath, payload) { const ps = fork(join(basePath, `${taskPath}.js`)) + + debug(payload.idx, payload) + return new Promise((resolver, rejecter) => { ps.send( payload ) ps.on('message', result => { @@ -71,7 +74,7 @@ function createTask(slots, taskPath, payloads, returnType = true) { * @return {promise} resolve the final result */ function splitTask(taskName, payload) { - // disable this for now @TODO + return Promise.resolve(false) const slots = getSlots() @@ -94,18 +97,16 @@ function splitTask(taskName, payload) { break; case 'pre': returnType = false; // default is array - scriptName = 'preprocess-resolver' + scriptName = 'preprocess-resolver'; // here we need to get a preprocessor, then we don't need to keep calling it // can not send a function as arument to the fork fn let { inDir, fileType, files } = payload; // debug(preprocessor, files) - payloads = files.map(file => ({ file, inDir, fileType, config: payload.config })) + payloads = files.map((file, idx) => ({ file, inDir, fileType, idx, config: payload.config })) postProcessor = result => { // should be running the fitler here - // return result.filter(obj => obj.ok) debug('postProcessor', result) - // run the filter here - return result; + return result.filter(obj => obj.ok) } break; default: diff --git a/packages/contract-cli/src/utils.js b/packages/contract-cli/src/utils.js index 71322c6c..2ce632b7 100644 --- a/packages/contract-cli/src/utils.js +++ b/packages/contract-cli/src/utils.js @@ -16,8 +16,6 @@ const MODULE_NAME = 'jsonql-contract' const getTimestamp = () => timestamp(true) const getDebug = (name) => debug(MODULE_NAME).extend(name) - - /** * check if there is a config file and use that value instead * @param {object} opts transformed diff --git a/packages/contract-cli/tests/split-task.test.js b/packages/contract-cli/tests/split-task.test.js index cb620b70..ee69038e 100644 --- a/packages/contract-cli/tests/split-task.test.js +++ b/packages/contract-cli/tests/split-task.test.js @@ -5,9 +5,12 @@ const { join } = require('path') const { chainPromises } = require('jsonql-utils') const os = require('os') const debug = require('debug')('jsonql-contract:test:split') +const { EXT } = require('jsonql-constants') + // import method frr testing const { getResolverFiles } = require('../src/generator/read-files-out-contract') const { splitTask } = require('../src/generator/split-task') +const { applyDefaultOptions } = require('../src') const { getResolver, getSourceType, @@ -40,7 +43,7 @@ test.before(t => { t.context.taskes = taskes }) -test.cb(`Should able to create multple array based on the cpu available`, t => { +test.cb(`Should able to create multiple array based on the cpu available`, t => { t.plan(1) debug(`available task slot is ${t.context.ctn}`) @@ -73,13 +76,32 @@ test.cb(`Should able to create multple array based on the cpu available`, t => { }) }) -test.cb(`It should able to split the task and return an array`, t => { - t.plan(1) - // need to get a config object - getResolverFiles(resolverDir) - .then(files => { - // prepare the payloads - t.truthy(files.length) - t.end() - }) +test.cb.skip(`It should able to split the task and return an array`, t => { + t.plan(2) + + applyDefaultOptions({ + resolverDir + }) + .then(config => { + return getResolverFiles(resolverDir) + .then(files => { + // prepare the payloads + t.truthy(files.length) + const payload = { + files, + config, + inDir: resolverDir, // we should take this option out later + fileType: EXT // this also should be taken out + } + return splitTask('pre', payload) + }) + .then(result => { + debug(result) + t.pass() + t.end() + }) + }) + + + }) -- Gitee From ad5d92adeda07f74fe52362c280ee5bfb0312be8 Mon Sep 17 00:00:00 2001 From: joelchu Date: Sun, 6 Oct 2019 12:09:14 +0800 Subject: [PATCH 2/4] test the new interface and working correctly --- packages/contract-cli/src/generator/process-file.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contract-cli/src/generator/process-file.js b/packages/contract-cli/src/generator/process-file.js index b2c87bf2..59da6d85 100644 --- a/packages/contract-cli/src/generator/process-file.js +++ b/packages/contract-cli/src/generator/process-file.js @@ -80,7 +80,7 @@ function preprocessResolverFile(files, inDir, fileType, config) { function processResolverFile(sourceType, files, config) { return callPostSplitTask(sourceType, files, config) .then(result => ( - result || _files + result || files .map(({ type, name, file, public, namespace }) => ( processResolverToContract(type, name, file, public, namespace, resourceType) )) -- Gitee From b92feab6250ca1c11e529c64a222cc03574611a5 Mon Sep 17 00:00:00 2001 From: joelchu Date: Sun, 6 Oct 2019 12:20:45 +0800 Subject: [PATCH 3/4] need to rethink about how to deal with the split test, the current form is really rubbish --- packages/contract-cli/package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contract-cli/package.json b/packages/contract-cli/package.json index ab8951e0..b9127ffa 100755 --- a/packages/contract-cli/package.json +++ b/packages/contract-cli/package.json @@ -1,6 +1,6 @@ { "name": "jsonql-contract", - "version": "1.7.19", + "version": "1.7.20", "description": "JS API / command line tool to generate the contract.json for jsonql", "main": "index.js", "files": [ -- Gitee From b41f6e9c992632658173f02cb342b21d3eab1b74 Mon Sep 17 00:00:00 2001 From: joelchu Date: Sun, 6 Oct 2019 13:13:03 +0800 Subject: [PATCH 4/4] completely strip down the split task interface --- .../src/generator/process-file.js | 54 +++++-------------- .../src/generator/read-files-out-contract.js | 15 +++--- 2 files changed, 21 insertions(+), 48 deletions(-) diff --git a/packages/contract-cli/src/generator/process-file.js b/packages/contract-cli/src/generator/process-file.js index 59da6d85..71af7531 100644 --- a/packages/contract-cli/src/generator/process-file.js +++ b/packages/contract-cli/src/generator/process-file.js @@ -7,33 +7,10 @@ const { getContractBase, processResolverToContract } = require('./get-resolver') -const { splitTask } = require('./split-task') -// const { logToFile } = require('./helpers') -// const { astParser, extractReturns, extractParams, isExpression, getJsdoc } = require('../ast') const { getDebug } = require('../utils') const debug = getDebug('process-file') // this was lost when using as api using let -/** - * wrapper method to call the pre splitTask - * - */ -function callPreSplitTask(inDir, fileType, files, config) { - return splitTask('pre', { inDir, fileType, files, config }) -} - -/** - * wrapper method to call the process splitTask - * - */ -function callPostSplitTask(_sourceType, _files, _config) { - return splitTask('process', { - sourceType: _sourceType, - files: _files, - config: _config - }) -} - /** * Take the map filter out to get the clean resolver objects * @param {array} _files files to process @@ -41,16 +18,12 @@ function callPostSplitTask(_sourceType, _files, _config) { * @return {array} resolver objects */ function processFilesAction(inDir, fileType, files, config) { - return callPreSplitTask(inDir, fileType, files, config) - .then(result => { - if (result === false) { - const preprocessor = getResolver(config, fileType) - return files.map( preprocessor ) - .filter( obj => obj.ok ) - .reduce( (n, m) => [...n, m], []) // do we need this? - } - return result; - }) + const preprocessor = getResolver(config, fileType) + return Promise.resolve( + files.map( preprocessor ) + .filter( obj => obj.ok ) + ) + // .reduce( (n, m) => [...n, m], []) } /** @@ -78,14 +51,13 @@ function preprocessResolverFile(files, inDir, fileType, config) { * @return {object} promise that resolve all the files key query / mutation */ function processResolverFile(sourceType, files, config) { - return callPostSplitTask(sourceType, files, config) - .then(result => ( - result || files - .map(({ type, name, file, public, namespace }) => ( - processResolverToContract(type, name, file, public, namespace, resourceType) - )) - .reduce(merge, getContractBase(sourceType)) - )) + return Promise.resolve( + files + .map(({ type, name, file, public, namespace }) => ( + processResolverToContract(type, name, file, public, namespace, resourceType) + )) + .reduce(merge, getContractBase(sourceType)) + ) } /** diff --git a/packages/contract-cli/src/generator/read-files-out-contract.js b/packages/contract-cli/src/generator/read-files-out-contract.js index 38638e60..4423f702 100644 --- a/packages/contract-cli/src/generator/read-files-out-contract.js +++ b/packages/contract-cli/src/generator/read-files-out-contract.js @@ -41,14 +41,15 @@ function readFilesOutContract(config) { const { resolverDir } = config; let fileType = config.ext || EXT; let timestart = Date.now() + return getResolverFiles(resolverDir, fileType) - .then(_files => ( - preprocessResolverFile(_files, resolverDir, fileType, config) - )) - .then(result => { - const [sourceType, files] = result; - return processResolverFile(sourceType, files, config) - .then(contract => [sourceType, contract]) + .then(_files => { + return preprocessResolverFile(_files, resolverDir, fileType, config) + .then(result => { + const [sourceType, files] = result; + return processResolverFile(sourceType, files, config) + .then(contract => [sourceType, contract]) + }) }) .then(result => { let timeend = Date.now() - timestart -- Gitee