From 35772db86a25bb37ec85173c9a5961ea20eb0778 Mon Sep 17 00:00:00 2001 From: Pirulax Date: Mon, 10 Jul 2023 23:31:14 +0200 Subject: [PATCH 01/16] Process file asynchronously --- index.js | 51 +++++++++++++++++++++++---------------------------- 1 file changed, 23 insertions(+), 28 deletions(-) diff --git a/index.js b/index.js index 2cf2255..198ec34 100644 --- a/index.js +++ b/index.js @@ -732,37 +732,32 @@ async function main() { throw new Error('No C/C++ files were found in the project that could be analyzed.'); } - // TODO: parallelism - const failedSourceFiles = []; - for (const command of analyzeCommands) { - const execOptions = { - cwd: buildDir, - env: command.env, - }; - - // TODO: timeouts - core.info(`Running analysis on: ${command.source}`); - try { - await exec.exec(`"${command.compiler}"`, command.args, execOptions); - } catch (err) { - core.debug(`Compilation failed with error: ${err}`); - core.debug("Environment:"); - core.debug(execOptions.env); - failedSourceFiles.push(command.source); - } - } - - if (failedSourceFiles.length > 0) { - const fileList = failedSourceFiles - .map(file => path.basename(file)) - .join(","); - throw new Error(`Analysis failed due to compiler errors in files: ${fileList}`); + // TODO: timeouts + let failed = [] + await Promise.all( + analyzeCommands.map(command => ( + async () => { + const execOptions = { + cwd: buildDir, + env: command.env, + } + try { + await exec.exec(`"${command.compiler}"`, command.args, execOptions); + } catch (err) { + core.info(`Compilation of ${command.source} failed with error: ${err}`); + core.info(`Environment: ${execOptions.env}`); + failed.push(command.source); + } + } + )) + ); + + if (failed.length > 0) { + throw new Error(`Analysis failed due to compiler errors in files: ${failed.map(file => path.basename(file)).join(",")}`); } - const sarifResults = analyzeCommands.map(command => command.sarifLog); - combineSarif(resultPath, sarifResults); + combineSarif(resultPath, analyzeCommands.map(command => command.sarifLog)); core.setOutput("sarif", resultPath); - } catch (error) { if (core.isDebug()) { core.setFailed(error.stack) From 3fcf3a51757d6d23c8f9663ac3277ff9abe07248 Mon Sep 17 00:00:00 2001 From: Pirulax Date: Mon, 10 Jul 2023 23:33:25 +0200 Subject: [PATCH 02/16] Don't print failed file list separetly --- index.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/index.js b/index.js index 198ec34..99ed27b 100644 --- a/index.js +++ b/index.js @@ -733,7 +733,6 @@ async function main() { } // TODO: timeouts - let failed = [] await Promise.all( analyzeCommands.map(command => ( async () => { @@ -746,16 +745,11 @@ async function main() { } catch (err) { core.info(`Compilation of ${command.source} failed with error: ${err}`); core.info(`Environment: ${execOptions.env}`); - failed.push(command.source); } } )) ); - if (failed.length > 0) { - throw new Error(`Analysis failed due to compiler errors in files: ${failed.map(file => path.basename(file)).join(",")}`); - } - combineSarif(resultPath, analyzeCommands.map(command => command.sarifLog)); core.setOutput("sarif", resultPath); } catch (error) { From 109b436635a1677651bc3055ee32dd72e183680e Mon Sep 17 00:00:00 2001 From: Pirulax Date: Mon, 10 Jul 2023 23:36:06 +0200 Subject: [PATCH 03/16] Throw exception directly in the promise body to stop the analysis of all other files --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index 99ed27b..88325c4 100644 --- a/index.js +++ b/index.js @@ -745,6 +745,7 @@ async function main() { } catch (err) { core.info(`Compilation of ${command.source} failed with error: ${err}`); core.info(`Environment: ${execOptions.env}`); + throw new Error(`Analysis failed due to compile errors in ${command.source}`) // No need to continue the analysis once a file has failed } } )) From 7f0b7e97a87a6d24252e42fb1e7f7b9810466bf8 Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:04:33 +0200 Subject: [PATCH 04/16] Update dist/index.js too --- dist/index.js | 50 ++++++++++++++++++++------------------------------ 1 file changed, 20 insertions(+), 30 deletions(-) diff --git a/dist/index.js b/dist/index.js index c7f44f9..bff49d1 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9348,37 +9348,27 @@ async function main() { throw new Error('No C/C++ files were found in the project that could be analyzed.'); } - // TODO: parallelism - const failedSourceFiles = []; - for (const command of analyzeCommands) { - const execOptions = { - cwd: buildDir, - env: command.env, - }; - - // TODO: timeouts - core.info(`Running analysis on: ${command.source}`); - try { - await exec.exec(`"${command.compiler}"`, command.args, execOptions); - } catch (err) { - core.debug(`Compilation failed with error: ${err}`); - core.debug("Environment:"); - core.debug(execOptions.env); - failedSourceFiles.push(command.source); - } - } - - if (failedSourceFiles.length > 0) { - const fileList = failedSourceFiles - .map(file => path.basename(file)) - .join(","); - throw new Error(`Analysis failed due to compiler errors in files: ${fileList}`); - } - - const sarifResults = analyzeCommands.map(command => command.sarifLog); - combineSarif(resultPath, sarifResults); + // TODO: timeouts + await Promise.all( + analyzeCommands.map(command => ( + async () => { + const execOptions = { + cwd: buildDir, + env: command.env, + } + try { + await exec.exec(`"${command.compiler}"`, command.args, execOptions); + } catch (err) { + core.info(`Compilation of ${command.source} failed with error: ${err}`); + core.info(`Environment: ${execOptions.env}`); + throw new Error(`Analysis failed due to compile errors in ${command.source}`) // No need to continue the analysis once a file has failed + } + } + )) + ); + + combineSarif(resultPath, analyzeCommands.map(command => command.sarifLog)); core.setOutput("sarif", resultPath); - } catch (error) { if (core.isDebug()) { core.setFailed(error.stack) From 4fffe32c3966b7f6e83aaba917c6aa6bbb8946d3 Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:20:16 +0200 Subject: [PATCH 05/16] Fix sarif --- dist/index.js | 5 +++++ index.js | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/dist/index.js b/dist/index.js index bff49d1..6ec42c5 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9348,6 +9348,8 @@ async function main() { throw new Error('No C/C++ files were found in the project that could be analyzed.'); } + core.info(`Running analysis on ${analyzeCommands.length} files`); + // TODO: timeouts await Promise.all( analyzeCommands.map(command => ( @@ -9367,7 +9369,10 @@ async function main() { )) ); + core.info("Combining sarif for all files"); combineSarif(resultPath, analyzeCommands.map(command => command.sarifLog)); + + core.info("Saving SARIF output"); core.setOutput("sarif", resultPath); } catch (error) { if (core.isDebug()) { diff --git a/index.js b/index.js index 88325c4..b362529 100644 --- a/index.js +++ b/index.js @@ -732,6 +732,8 @@ async function main() { throw new Error('No C/C++ files were found in the project that could be analyzed.'); } + core.info(`Running analysis on ${analyzeCommands.length} files`) + // TODO: timeouts await Promise.all( analyzeCommands.map(command => ( @@ -751,7 +753,10 @@ async function main() { )) ); + core.info("Combining SARIF for all files") combineSarif(resultPath, analyzeCommands.map(command => command.sarifLog)); + + core.info("Saving SARIF output"); core.setOutput("sarif", resultPath); } catch (error) { if (core.isDebug()) { From 127ab62101a6abe53e4ec222a10a819099b57bbe Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:37:23 +0200 Subject: [PATCH 06/16] run test From c913400cf2ab4741cbb8b32e7414182c89fe88eb Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:42:01 +0200 Subject: [PATCH 07/16] Print additional info --- dist/index.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/dist/index.js b/dist/index.js index 6ec42c5..b032568 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9352,21 +9352,22 @@ async function main() { // TODO: timeouts await Promise.all( - analyzeCommands.map(command => ( - async () => { + analyzeCommands.map(cmd => { + return async () => { + core.info(`Running analysis on ${cmd.source}`) const execOptions = { cwd: buildDir, - env: command.env, + env: cmd.env, } try { - await exec.exec(`"${command.compiler}"`, command.args, execOptions); + await exec.exec(`"${cmd.compiler}"`, cmd.args, execOptions); } catch (err) { - core.info(`Compilation of ${command.source} failed with error: ${err}`); + core.info(`Compilation of ${cmd.source} failed with error: ${err}`); core.info(`Environment: ${execOptions.env}`); - throw new Error(`Analysis failed due to compile errors in ${command.source}`) // No need to continue the analysis once a file has failed + throw new Error(`Analysis failed due to compile errors in ${cmd.source}`) // No need to continue the analysis once a file has failed } } - )) + }) ); core.info("Combining sarif for all files"); From 3a3f7157ce905c37f1e695f3e8fab59c93a96dbc Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:45:47 +0200 Subject: [PATCH 08/16] Fix --- dist/index.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dist/index.js b/dist/index.js index b032568..bc45161 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9352,8 +9352,8 @@ async function main() { // TODO: timeouts await Promise.all( - analyzeCommands.map(cmd => { - return async () => { + analyzeCommands.map(cmd => ( + (async () => { core.info(`Running analysis on ${cmd.source}`) const execOptions = { cwd: buildDir, @@ -9366,8 +9366,8 @@ async function main() { core.info(`Environment: ${execOptions.env}`); throw new Error(`Analysis failed due to compile errors in ${cmd.source}`) // No need to continue the analysis once a file has failed } - } - }) + })() + )) ); core.info("Combining sarif for all files"); From 80f8071599339f12195311f44b761e9123d4a406 Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:50:53 +0200 Subject: [PATCH 09/16] Use JSON.stringify for `env` --- dist/index.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/dist/index.js b/dist/index.js index bc45161..2f778e7 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9354,7 +9354,6 @@ async function main() { await Promise.all( analyzeCommands.map(cmd => ( (async () => { - core.info(`Running analysis on ${cmd.source}`) const execOptions = { cwd: buildDir, env: cmd.env, @@ -9363,8 +9362,8 @@ async function main() { await exec.exec(`"${cmd.compiler}"`, cmd.args, execOptions); } catch (err) { core.info(`Compilation of ${cmd.source} failed with error: ${err}`); - core.info(`Environment: ${execOptions.env}`); - throw new Error(`Analysis failed due to compile errors in ${cmd.source}`) // No need to continue the analysis once a file has failed + core.info(`Environment: ${JSON.stringify(execOptions.env)}`); + throw new Error(`Analysis failed due to compile errors in ${cmd.source}`) } })() )) From 63dc60b9f2e781e6bdb6d6a0976969d9eaca9ef7 Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:51:12 +0200 Subject: [PATCH 10/16] Fix grammer --- dist/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/index.js b/dist/index.js index 2f778e7..878e051 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9372,7 +9372,7 @@ async function main() { core.info("Combining sarif for all files"); combineSarif(resultPath, analyzeCommands.map(command => command.sarifLog)); - core.info("Saving SARIF output"); + core.info("Save SARIF output"); core.setOutput("sarif", resultPath); } catch (error) { if (core.isDebug()) { From 9798c8ec4d5bf29f8d3b88a027de6b7fdd0593d2 Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:53:57 +0200 Subject: [PATCH 11/16] pretty-print json.stringify --- dist/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/index.js b/dist/index.js index 878e051..dadf6a9 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9362,7 +9362,7 @@ async function main() { await exec.exec(`"${cmd.compiler}"`, cmd.args, execOptions); } catch (err) { core.info(`Compilation of ${cmd.source} failed with error: ${err}`); - core.info(`Environment: ${JSON.stringify(execOptions.env)}`); + core.info(`Environment: ${JSON.stringify(execOptions.env, null, 4)}`); throw new Error(`Analysis failed due to compile errors in ${cmd.source}`) } })() From e4b5ff8754ffcc155d1a9fe1e1d70271b885531f Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 00:55:27 +0200 Subject: [PATCH 12/16] Fix grammar --- dist/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/index.js b/dist/index.js index dadf6a9..5a5204d 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9363,7 +9363,7 @@ async function main() { } catch (err) { core.info(`Compilation of ${cmd.source} failed with error: ${err}`); core.info(`Environment: ${JSON.stringify(execOptions.env, null, 4)}`); - throw new Error(`Analysis failed due to compile errors in ${cmd.source}`) + throw new Error(`Analysis failed due to errors in while trying to compile ${cmd.source}`) } })() )) From ed63abf7ca544e9872dd8de88e1a8fb02c92972f Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 01:14:15 +0200 Subject: [PATCH 13/16] Run less jobs in parallel --- dist/index.js | 40 +++++++++++++++++++++++----------------- 1 file changed, 23 insertions(+), 17 deletions(-) diff --git a/dist/index.js b/dist/index.js index 5a5204d..6479a18 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9351,23 +9351,29 @@ async function main() { core.info(`Running analysis on ${analyzeCommands.length} files`); // TODO: timeouts - await Promise.all( - analyzeCommands.map(cmd => ( - (async () => { - const execOptions = { - cwd: buildDir, - env: cmd.env, - } - try { - await exec.exec(`"${cmd.compiler}"`, cmd.args, execOptions); - } catch (err) { - core.info(`Compilation of ${cmd.source} failed with error: ${err}`); - core.info(`Environment: ${JSON.stringify(execOptions.env, null, 4)}`); - throw new Error(`Analysis failed due to errors in while trying to compile ${cmd.source}`) - } - })() - )) - ); + // We have to process in chunks, otherwise we'll run into out-of-memory situations + // generally [I believe] it makes no sense to run more "parallel" jobs than the number of cpu threads + // TODO: Perhaps use `os.cpus()` to get the cpu thread count? + const CHUNK_SIZE = 8; + for (let i = 0; i < analyzeCommands.length; i += CHUNK_SIZE) { + await Promise.all( + analyzeCommands.splice(i, Math.min(i + CHUNK_SIZE, analyzeCommands.length)).map(cmd => ( + (async () => { + const execOptions = { + cwd: buildDir, + env: cmd.env, + } + try { + await exec.exec(`"${cmd.compiler}"`, cmd.args, execOptions); + } catch (err) { + core.info(`Compilation of ${cmd.source} failed with error: ${err}`); + core.info(`Environment: ${JSON.stringify(execOptions.env, null, 4)}`); + throw new Error(`Analysis failed due to errors in while trying to compile ${cmd.source}`) + } + })() + )) + ); + } core.info("Combining sarif for all files"); combineSarif(resultPath, analyzeCommands.map(command => command.sarifLog)); From 8961c32b4de2ed694ac6b24aefa2f2e03fe658ee Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 01:25:34 +0200 Subject: [PATCH 14/16] Fix pch error --- dist/index.js | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/dist/index.js b/dist/index.js index 6479a18..8da0ae7 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9350,28 +9350,35 @@ async function main() { core.info(`Running analysis on ${analyzeCommands.length} files`); + async function processCommand(cmd) { + const execOptions = { + cwd: buildDir, + env: cmd.env, + } + try { + await exec.exec(`"${cmd.compiler}"`, cmd.args, execOptions); + } catch (err) { + core.info(`Compilation of ${cmd.source} failed with error: ${err}`); + core.info(`Environment: ${JSON.stringify(execOptions.env, null, 4)}`); + throw new Error(`Analysis failed due to errors in while trying to compile ${cmd.source}`) + } + } + // TODO: timeouts + + // First file is the pch - If there's no pch, it's going to be a regular file + // It has to be compiled separately, as all other files require it [and a "Permission Denioed" error will be raised if they try to access it] + await processCommand(createAnalysisCommands[0]) + // We have to process in chunks, otherwise we'll run into out-of-memory situations // generally [I believe] it makes no sense to run more "parallel" jobs than the number of cpu threads // TODO: Perhaps use `os.cpus()` to get the cpu thread count? const CHUNK_SIZE = 8; for (let i = 0; i < analyzeCommands.length; i += CHUNK_SIZE) { await Promise.all( - analyzeCommands.splice(i, Math.min(i + CHUNK_SIZE, analyzeCommands.length)).map(cmd => ( - (async () => { - const execOptions = { - cwd: buildDir, - env: cmd.env, - } - try { - await exec.exec(`"${cmd.compiler}"`, cmd.args, execOptions); - } catch (err) { - core.info(`Compilation of ${cmd.source} failed with error: ${err}`); - core.info(`Environment: ${JSON.stringify(execOptions.env, null, 4)}`); - throw new Error(`Analysis failed due to errors in while trying to compile ${cmd.source}`) - } - })() - )) + analyzeCommands + .slice(i, Math.min(i + CHUNK_SIZE, analyzeCommands.length)) + .map(cmd => processCommand(cmd)) ); } From d25968a9e3bc1835e3d9f81414179d3b9f8aebbe Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 01:32:16 +0200 Subject: [PATCH 15/16] Fix code --- dist/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dist/index.js b/dist/index.js index 8da0ae7..ee81729 100644 --- a/dist/index.js +++ b/dist/index.js @@ -9368,7 +9368,7 @@ async function main() { // First file is the pch - If there's no pch, it's going to be a regular file // It has to be compiled separately, as all other files require it [and a "Permission Denioed" error will be raised if they try to access it] - await processCommand(createAnalysisCommands[0]) + await processCommand(analyzeCommands[0]) // We have to process in chunks, otherwise we'll run into out-of-memory situations // generally [I believe] it makes no sense to run more "parallel" jobs than the number of cpu threads From 8087e57d7d940b441e4745ce5c79e33bf40e83bc Mon Sep 17 00:00:00 2001 From: Pirulax Date: Tue, 11 Jul 2023 01:49:01 +0200 Subject: [PATCH 16/16] copy code to index.js --- index.js | 54 ++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/index.js b/index.js index b362529..0c6c156 100644 --- a/index.js +++ b/index.js @@ -732,31 +732,44 @@ async function main() { throw new Error('No C/C++ files were found in the project that could be analyzed.'); } - core.info(`Running analysis on ${analyzeCommands.length} files`) + core.info(`Running analysis on ${analyzeCommands.length} files`); + + async function processCommand(cmd) { + const execOptions = { + cwd: buildDir, + env: cmd.env, + } + try { + await exec.exec(`"${cmd.compiler}"`, cmd.args, execOptions); + } catch (err) { + core.info(`Compilation of ${cmd.source} failed with error: ${err}`); + core.info(`Environment: ${JSON.stringify(execOptions.env, null, 4)}`); + throw new Error(`Analysis failed due to errors in while trying to compile ${cmd.source}`) + } + } // TODO: timeouts - await Promise.all( - analyzeCommands.map(command => ( - async () => { - const execOptions = { - cwd: buildDir, - env: command.env, - } - try { - await exec.exec(`"${command.compiler}"`, command.args, execOptions); - } catch (err) { - core.info(`Compilation of ${command.source} failed with error: ${err}`); - core.info(`Environment: ${execOptions.env}`); - throw new Error(`Analysis failed due to compile errors in ${command.source}`) // No need to continue the analysis once a file has failed - } - } - )) - ); + + // First file is the pch - If there's no pch, it's going to be a regular file + // It has to be compiled separately, as all other files require it [and a "Permission Denioed" error will be raised if they try to access it] + await processCommand(analyzeCommands[0]) - core.info("Combining SARIF for all files") + // We have to process in chunks, otherwise we'll run into out-of-memory situations + // generally [I believe] it makes no sense to run more "parallel" jobs than the number of cpu threads + // TODO: Perhaps use `os.cpus()` to get the cpu thread count? + const CHUNK_SIZE = 8; + for (let i = 0; i < analyzeCommands.length; i += CHUNK_SIZE) { + await Promise.all( + analyzeCommands + .slice(i, Math.min(i + CHUNK_SIZE, analyzeCommands.length)) + .map(cmd => processCommand(cmd)) + ); + } + + core.info("Combining sarif for all files"); combineSarif(resultPath, analyzeCommands.map(command => command.sarifLog)); - core.info("Saving SARIF output"); + core.info("Save SARIF output"); core.setOutput("sarif", resultPath); } catch (error) { if (core.isDebug()) { @@ -771,6 +784,7 @@ async function main() { } } + if (require.main === module) { (async () => { await main();