Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Load package.json on init-linter #4363

Merged
merged 36 commits into from
Nov 13, 2023
Merged
Show file tree
Hide file tree
Changes from 29 commits
Commits
Show all changes
36 commits
Select commit Hold shift + click to select a range
2ff0e11
wip
vdiez Nov 7, 2023
c99f81a
add tests
vdiez Nov 8, 2023
0d6049f
add tests for analysis and build phases
vdiez Nov 9, 2023
25362ca
simplified test accessing packageJson from rule
vdiez Nov 9, 2023
8cd93cc
normalize baseDir before package.json search
vdiez Nov 9, 2023
77f1dfd
remove bundleDependencies
vdiez Nov 9, 2023
8c8142b
normalize path before getting package.json for file
vdiez Nov 9, 2023
b86ce13
Changes from review
vdiez Nov 9, 2023
49891bd
Update packages/jsts/src/dependencies/package-json/project-package-js…
vdiez Nov 9, 2023
3c563bc
Update packages/jsts/src/dependencies/package-json/project-package-js…
vdiez Nov 9, 2023
20cc61a
Update packages/jsts/tests/analysis/analyzer.test.ts
vdiez Nov 9, 2023
c4956fb
Update packages/jsts/tests/analysis/analyzer.test.ts
vdiez Nov 9, 2023
a7fef37
Update packages/jsts/tests/analysis/analyzer.test.ts
vdiez Nov 9, 2023
7389e33
Update packages/jsts/tests/analysis/analyzer.test.ts
vdiez Nov 9, 2023
9d776b1
changes after review
vdiez Nov 9, 2023
d2d9ee5
add test for Vue
vdiez Nov 9, 2023
f06b1e6
change method name
vdiez Nov 9, 2023
9677aef
fix tests
vdiez Nov 9, 2023
2604103
pass ignored patterns from Java
vdiez Nov 9, 2023
6fc3651
fix tests
vdiez Nov 10, 2023
ec702c6
remove ArrayUtils
vdiez Nov 10, 2023
adcc366
fix qa
vdiez Nov 10, 2023
5594e7c
fix test
vdiez Nov 10, 2023
531a9eb
enable debug logs
vdiez Nov 10, 2023
fb9e882
enable debug logs
vdiez Nov 10, 2023
dfdbff7
changes from review
vdiez Nov 10, 2023
eeab461
fix shadowing
vdiez Nov 10, 2023
afbb9e9
remove logs
vdiez Nov 10, 2023
bb1e7ad
remove logs
vdiez Nov 10, 2023
28ceea5
improve coverage
vdiez Nov 10, 2023
c7982d1
typo
vdiez Nov 10, 2023
5c5ef4d
refactor exclusions
vdiez Nov 10, 2023
9a6ff6c
filter empty strings
vdiez Nov 10, 2023
c7c3659
allow overriding config to empty exclusions
vdiez Nov 10, 2023
b8eebe3
remove unused imports
vdiez Nov 10, 2023
f1d25eb
fix SQ issue
vdiez Nov 10, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
450 changes: 382 additions & 68 deletions package-lock.json

Large diffs are not rendered by default.

3 changes: 3 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,7 @@
"htmlparser2": "9.0.0",
"jsx-ast-utils": "3.3.5",
"lodash.clone": "4.5.0",
"minimatch": "^9.0.3",
"module-alias": "2.2.3",
"postcss": "8.4.31",
"postcss-html": "0.36.0",
Expand All @@ -102,6 +103,7 @@
"scslre": "0.2.0",
"stylelint": "15.10.0",
"tmp": "0.2.1",
"type-fest": "4.6.0",
vdiez marked this conversation as resolved.
Show resolved Hide resolved
"typescript": "5.1.6",
"vue-eslint-parser": "9.3.0",
"yaml": "2.3.1"
Expand Down Expand Up @@ -129,6 +131,7 @@
"htmlparser2",
"jsx-ast-utils",
"lodash.clone",
"minimatch",
"module-alias",
"postcss",
"postcss-html",
Expand Down
4 changes: 3 additions & 1 deletion packages/bridge/src/worker.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ const {
deleteProgram,
initializeLinter,
writeTSConfigFile,
searchPackageJsonFiles,
} = require('@sonar/jsts');
const { readFile, setContext } = require('@sonar/shared/helpers');
const { analyzeCSS } = require('@sonar/css');
Expand Down Expand Up @@ -136,8 +137,9 @@ if (parentPort) {
}

case 'on-init-linter': {
const { rules, environments, globals, linterId } = data;
const { rules, environments, globals, linterId, baseDir, exclusions } = data;
initializeLinter(rules, environments, globals, linterId);
await searchPackageJsonFiles(baseDir, exclusions);
parentThread.postMessage({ type: 'success', result: 'OK!' });
break;
}
Expand Down
5 changes: 5 additions & 0 deletions packages/jsts/src/builders/build.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import { JsTsAnalysisInput } from '../analysis';
import { buildParserOptions, parseForESLint, parsers } from '../parsers';
import { getProgramById } from '../program';
import { Linter } from 'eslint';
import { getNearestPackageJson } from '../dependencies';

/**
* Builds an ESLint SourceCode for JavaScript / TypeScript
Expand All @@ -35,6 +36,7 @@ import { Linter } from 'eslint';
*/
export function buildSourceCode(input: JsTsAnalysisInput, language: JsTsLanguage) {
const vueFile = isVueFile(input.filePath);
const packageJson = getNearestPackageJson(input.filePath);

if (shouldUseTypescriptParser(language)) {
const options: Linter.ParserOptions = {
Expand All @@ -51,6 +53,7 @@ export function buildSourceCode(input: JsTsAnalysisInput, language: JsTsLanguage
input.fileContent,
vueFile ? parsers.vuejs.parse : parsers.typescript.parse,
buildParserOptions(options, false),
packageJson?.contents,
);
} catch (error) {
debug(`Failed to parse ${input.filePath} with TypeScript parser: ${error.message}`);
Expand All @@ -66,6 +69,7 @@ export function buildSourceCode(input: JsTsAnalysisInput, language: JsTsLanguage
input.fileContent,
vueFile ? parsers.vuejs.parse : parsers.javascript.parse,
buildParserOptions({ parser: vueFile ? parsers.javascript.parser : undefined }, true),
packageJson?.contents,
);
} catch (error) {
debug(`Failed to parse ${input.filePath} with Javascript parser: ${error.message}`);
Expand All @@ -80,6 +84,7 @@ export function buildSourceCode(input: JsTsAnalysisInput, language: JsTsLanguage
input.fileContent,
parsers.javascript.parse,
buildParserOptions({ sourceType: 'script' }, true),
packageJson?.contents,
);
} catch (error) {
debug(
Expand Down
20 changes: 20 additions & 0 deletions packages/jsts/src/dependencies/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
/*
* SonarQube JavaScript Plugin
* Copyright (C) 2011-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
export * from './package-json';
37 changes: 37 additions & 0 deletions packages/jsts/src/dependencies/package-json/index.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
/*
vdiez marked this conversation as resolved.
Show resolved Hide resolved
* SonarQube JavaScript Plugin
* Copyright (C) 2011-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/

import { PackageJsons } from './project-package-json';

const PackageJsonsByBaseDir = new PackageJsons();

async function searchPackageJsonFiles(baseDir: string, exclusions: string[]) {
await PackageJsonsByBaseDir.searchPackageJsonFiles(baseDir, exclusions);
}

function getNearestPackageJson(file: string) {
return PackageJsonsByBaseDir.getPackageJsonForFile(file);
}

function getAllPackageJsons() {
return PackageJsonsByBaseDir.db;
}

export { searchPackageJsonFiles, getNearestPackageJson, getAllPackageJsons };
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/*
* SonarQube JavaScript Plugin
* Copyright (C) 2011-2023 SonarSource SA
* mailto:info AT sonarsource DOT com
*
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 3 of the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public License
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
import fs from 'fs/promises';
import path from 'path';
import { toUnixPath, debug, error, readFile } from '@sonar/shared/helpers';
import { PackageJson as PJ } from 'type-fest';
import { Minimatch } from 'minimatch';

const PACKAGE_JSON = 'package.json';

// Patterns enforced to be ignored no matter what the user configures on sonar.properties
const IGNORED_PATTERNS = ['**/.scannerwork/**'];

export interface PackageJson {
filename: string;
contents: PJ;
}

export class PackageJsons {
readonly db: Map<string, PackageJson> = new Map();

/**
* Look for package.json files in a given path and its child paths.
* node_modules is ignored
*
* @param dir parent folder where the search starts
* @param exclusions glob patterns to ignore while walking the tree
*/
async searchPackageJsonFiles(dir: string, exclusions: string[]) {
try {
const patterns = exclusions
.concat(IGNORED_PATTERNS)
.map(exclusion => new Minimatch(exclusion));
await this.walkDirectory(path.posix.normalize(toUnixPath(dir)), patterns);
} catch (e) {
error(`Error while searching for package.json files: ${e}`);
vdiez marked this conversation as resolved.
Show resolved Hide resolved
}
}

private async walkDirectory(dir: string, ignoredPatterns: Minimatch[]) {
const files = await fs.readdir(dir, { withFileTypes: true });
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why do you need to await the result here, if function is async?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

because otherwise files is a promise and would not be able to iterate

for (const file of files) {
const filename = path.posix.join(dir, file.name);
if (ignoredPatterns.some(pattern => pattern.match(filename))) {
continue; // is ignored pattern
}
if (file.isDirectory()) {
await this.walkDirectory(filename, ignoredPatterns);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why await?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

not needed but to keep order of execution

} else if (file.name.toLowerCase() === PACKAGE_JSON && !file.isDirectory()) {
try {
debug(`Found package.json: ${filename}`);
const contents = JSON.parse(await readFile(filename));
this.db.set(dir, { filename, contents });
} catch (e) {
debug(`Error reading file ${filename}: ${e}`);
}
}
}
vdiez marked this conversation as resolved.
Show resolved Hide resolved
}
/**
* Given a filename, find the nearest package.json
*
* @param file source file for which we need a package.json
*/
getPackageJsonForFile(file: string) {
vdiez marked this conversation as resolved.
Show resolved Hide resolved
if (this.db.size === 0) {
return null;
}
let currentDir = path.posix.dirname(path.posix.normalize(toUnixPath(file)));
vdiez marked this conversation as resolved.
Show resolved Hide resolved
do {
const packageJson = this.db.get(currentDir);
if (packageJson) {
return packageJson;
}
currentDir = path.posix.dirname(currentDir);
} while (currentDir !== path.posix.dirname(currentDir));
return null;
vdiez marked this conversation as resolved.
Show resolved Hide resolved
}
}
1 change: 1 addition & 0 deletions packages/jsts/src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
*/
export * from './analysis';
export * from './builders';
export * from './dependencies';
export * from './linter';
export * from './parsers';
export * from './program';
Expand Down
13 changes: 11 additions & 2 deletions packages/jsts/src/parsers/parse.ts
Original file line number Diff line number Diff line change
Expand Up @@ -20,21 +20,30 @@
import { APIError } from '@sonar/shared/errors';
import { SourceCode } from 'eslint';
import { ParseFunction } from './eslint';
import { PackageJson } from 'type-fest';

/**
* Parses a JavaScript / TypeScript analysis input with an ESLint-based parser
* @param code the JavaScript / TypeScript code to parse
* @param parse the ESLint parsing function to use for parsing
* @param options the ESLint parser options
* @param packageJson package.json contents containing dependencies
* @returns the parsed source code
*/
export function parseForESLint(code: string, parse: ParseFunction, options: {}): SourceCode {
export function parseForESLint(
code: string,
parse: ParseFunction,
options: {},
packageJson?: PackageJson,
): SourceCode {
try {
const result = parse(code, options);
const parserServices = result.services || {};
parserServices.packageJson = packageJson;
vdiez marked this conversation as resolved.
Show resolved Hide resolved
return new SourceCode({
...result,
text: code,
parserServices: result.services,
parserServices,
});
} catch ({ lineNumber, message }) {
if (message.startsWith('Debug Failure')) {
Expand Down
53 changes: 51 additions & 2 deletions packages/jsts/tests/analysis/analyzer.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -25,12 +25,14 @@ import {
analyzeJSTS,
JsTsAnalysisOutput,
createAndSaveProgram,
searchPackageJsonFiles,
} from '../../src';
import { APIError } from '@sonar/shared/errors';
import { jsTsInput } from '../tools';

import { jsTsInput, parseJavaScriptSourceFile } from '../tools';
import { Linter, Rule } from 'eslint';
describe('analyzeJSTS', () => {
beforeEach(() => {
jest.resetModules();
setContext({
workDir: '/tmp/dir',
shouldUseTypeScriptParserForJS: false,
Expand Down Expand Up @@ -893,4 +895,51 @@ describe('analyzeJSTS', () => {
APIError.parsingError('Unexpected token (3:0)', { line: 3 }),
);
});

it('package.json should be available in rule context', async () => {
const baseDir = path.join(__dirname, 'fixtures', 'package-json');
await searchPackageJsonFiles(baseDir, []);

const linter = new Linter();
linter.defineRule('custom-rule-file', {
create(context) {
return {
CallExpression(node) {
expect(context.parserServices.packageJson).toBeDefined();
expect(context.parserServices.packageJson.name).toEqual('test-module');
context.report({
node: node.callee,
message: 'call',
});
},
};
},
} as Rule.RuleModule);

const filePath = path.join(baseDir, 'custom.js');
const sourceCode = await parseJavaScriptSourceFile(filePath);
expect(sourceCode.parserServices.packageJson).toBeDefined();
expect(sourceCode.parserServices.packageJson.name).toEqual('test-module');

const issues = linter.verify(
sourceCode,
{ rules: { 'custom-rule-file': 'error' } },
{ filename: filePath, allowInlineConfig: false },
);
expect(issues).toHaveLength(1);
expect(issues[0].message).toEqual('call');

const vueFilePath = path.join(baseDir, 'custom.js');
vdiez marked this conversation as resolved.
Show resolved Hide resolved
const vueSourceCode = await parseJavaScriptSourceFile(vueFilePath);
expect(vueSourceCode.parserServices.packageJson).toBeDefined();
expect(vueSourceCode.parserServices.packageJson.name).toEqual('test-module');

const vueIssues = linter.verify(
vueSourceCode,
{ rules: { 'custom-rule-file': 'error' } },
{ filename: vueFilePath, allowInlineConfig: false },
);
expect(vueIssues).toHaveLength(1);
expect(vueIssues[0].message).toEqual('call');
});
});
3 changes: 3 additions & 0 deletions packages/jsts/tests/analysis/fixtures/package-json/code.vue
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
<script>
foo()
</script>
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
foo()
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"name": "test-module",
"version": "1.0.0",
"author": "Your Name <[email protected]>"
}
19 changes: 17 additions & 2 deletions packages/jsts/tests/builders/build.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
* along with this program; if not, write to the Free Software Foundation,
* Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
import { setContext } from '@sonar/shared/helpers';
import { buildSourceCode } from '../../src';
import { setContext, toUnixPath } from '@sonar/shared/helpers';
import { buildSourceCode, searchPackageJsonFiles } from '../../src';
import path from 'path';
import { AST } from 'vue-eslint-parser';
import { jsTsInput } from '../tools';
Expand Down Expand Up @@ -289,4 +289,19 @@ describe('buildSourceCode', () => {
const log = `DEBUG Failed to parse ${filePath} with TypeScript parser: Expression expected.`;
expect(console.log).toHaveBeenCalledWith(log);
});

it('should include package.json contents in SourceCode', async () => {
console.log = jest.fn();
const baseDir = path.join(__dirname, 'fixtures', 'build');
const filePath = path.join(baseDir, 'file.ts');
const packageJson = path.join(baseDir, 'package.json');
await searchPackageJsonFiles(baseDir, []);
const log = `DEBUG Found package.json: ${toUnixPath(packageJson)}`;
expect(console.log).toHaveBeenCalledWith(log);

const result = buildSourceCode(await jsTsInput({ filePath }), 'ts');

expect(result.parserServices.packageJson).toBeDefined();
expect(result.parserServices.packageJson.name).toEqual('test-build-module');
});
});
5 changes: 5 additions & 0 deletions packages/jsts/tests/builders/fixtures/build/package.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
{
"name": "test-build-module",
"version": "1.0.0",
"author": "Your Name <[email protected]>"
}
Loading