Skip to content

Commit 97af137

Browse files
koesie10koesie10
authored
Merge pull request #2263 from github/koesie10/data-extensions-editor-calls
Show external API calls in data extensions editor
2 parents 5d7d1b2 + c4f6155 commit 97af137

12 files changed

Lines changed: 1074 additions & 7 deletions

File tree

‎extensions/ql-vscode/src/data-extensions-editor/bqrs.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
import { DecodedBqrsChunk } from "../pure/bqrs-cli-types";
2+
import { Call, ExternalApiUsage } from "./external-api-usage";
3+
4+
export function decodeBqrsToExternalApiUsages(
5+
chunk: DecodedBqrsChunk,
6+
): ExternalApiUsage[] {
7+
const methodsByApiName = new Map<string, ExternalApiUsage>();
8+
9+
chunk?.tuples.forEach((tuple) => {
10+
const signature = tuple[0] as string;
11+
const supported = tuple[1] as boolean;
12+
const usage = tuple[2] as Call;
13+
14+
const [packageWithType, methodDeclaration] = signature.split("#");
15+
16+
const packageName = packageWithType.substring(
17+
0,
18+
packageWithType.lastIndexOf("."),
19+
);
20+
const typeName = packageWithType.substring(
21+
packageWithType.lastIndexOf(".") + 1,
22+
);
23+
24+
const methodName = methodDeclaration.substring(
25+
0,
26+
methodDeclaration.indexOf("("),
27+
);
28+
const methodParameters = methodDeclaration.substring(
29+
methodDeclaration.indexOf("("),
30+
);
31+
32+
if (!methodsByApiName.has(signature)) {
33+
methodsByApiName.set(signature, {
34+
signature,
35+
packageName,
36+
typeName,
37+
methodName,
38+
methodParameters,
39+
supported,
40+
usages: [],
41+
});
42+
}
43+
44+
const method = methodsByApiName.get(signature)!;
45+
method.usages.push(usage);
46+
});
47+
48+
const externalApiUsages = Array.from(methodsByApiName.values());
49+
externalApiUsages.sort((a, b) => {
50+
// Sort by number of usages descending
51+
return b.usages.length - a.usages.length;
52+
});
53+
return externalApiUsages;
54+
}

‎extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts‎

Lines changed: 56 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,70 @@
11
import { ExtensionContext } from "vscode";
22
import { DataExtensionsEditorView } from "./data-extensions-editor-view";
33
import { DataExtensionsEditorCommands } from "../common/commands";
4+
import { CodeQLCliServer } from "../cli";
5+
import { QueryRunner } from "../queryRunner";
6+
import { DatabaseManager } from "../local-databases";
7+
import { extLogger } from "../common";
8+
import { ensureDir } from "fs-extra";
9+
import { join } from "path";
410

511
export class DataExtensionsEditorModule {
6-
public constructor(private readonly ctx: ExtensionContext) {}
12+
private readonly queryStorageDir: string;
13+
14+
private constructor(
15+
private readonly ctx: ExtensionContext,
16+
private readonly databaseManager: DatabaseManager,
17+
private readonly cliServer: CodeQLCliServer,
18+
private readonly queryRunner: QueryRunner,
19+
baseQueryStorageDir: string,
20+
) {
21+
this.queryStorageDir = join(
22+
baseQueryStorageDir,
23+
"data-extensions-editor-results",
24+
);
25+
}
26+
27+
public static async initialize(
28+
ctx: ExtensionContext,
29+
databaseManager: DatabaseManager,
30+
cliServer: CodeQLCliServer,
31+
queryRunner: QueryRunner,
32+
queryStorageDir: string,
33+
): Promise<DataExtensionsEditorModule> {
34+
const dataExtensionsEditorModule = new DataExtensionsEditorModule(
35+
ctx,
36+
databaseManager,
37+
cliServer,
38+
queryRunner,
39+
queryStorageDir,
40+
);
41+
42+
await dataExtensionsEditorModule.initialize();
43+
return dataExtensionsEditorModule;
44+
}
745

846
public getCommands(): DataExtensionsEditorCommands {
947
return {
1048
"codeQL.openDataExtensionsEditor": async () => {
11-
const view = new DataExtensionsEditorView(this.ctx);
49+
const db = this.databaseManager.currentDatabaseItem;
50+
if (!db) {
51+
void extLogger.log("No database selected");
52+
return;
53+
}
54+
55+
const view = new DataExtensionsEditorView(
56+
this.ctx,
57+
this.cliServer,
58+
this.queryRunner,
59+
this.queryStorageDir,
60+
db,
61+
);
1262
await view.openView();
1363
},
1464
};
1565
}
66+
67+
private async initialize(): Promise<void> {
68+
await ensureDir(this.queryStorageDir);
69+
}
1670
}

‎extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts‎

Lines changed: 173 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,37 @@
1-
import { ExtensionContext, ViewColumn } from "vscode";
1+
import { CancellationTokenSource, ExtensionContext, ViewColumn } from "vscode";
22
import { AbstractWebview, WebviewPanelConfig } from "../abstract-webview";
33
import {
44
FromDataExtensionsEditorMessage,
55
ToDataExtensionsEditorMessage,
66
} from "../pure/interface-types";
7+
import { ProgressUpdate } from "../progress";
8+
import { extLogger, TeeLogger } from "../common";
9+
import { CoreCompletedQuery, QueryRunner } from "../queryRunner";
10+
import { qlpackOfDatabase } from "../contextual/queryResolver";
11+
import { file } from "tmp-promise";
12+
import { writeFile } from "fs-extra";
13+
import { dump } from "js-yaml";
14+
import {
15+
getOnDiskWorkspaceFolders,
16+
showAndLogExceptionWithTelemetry,
17+
} from "../helpers";
18+
import { DatabaseItem } from "../local-databases";
19+
import { CodeQLCliServer } from "../cli";
20+
import { decodeBqrsToExternalApiUsages } from "./bqrs";
21+
import { redactableError } from "../pure/errors";
22+
import { asError, getErrorMessage } from "../pure/helpers-pure";
723

824
export class DataExtensionsEditorView extends AbstractWebview<
925
ToDataExtensionsEditorMessage,
1026
FromDataExtensionsEditorMessage
1127
> {
12-
public constructor(ctx: ExtensionContext) {
28+
public constructor(
29+
ctx: ExtensionContext,
30+
private readonly cliServer: CodeQLCliServer,
31+
private readonly queryRunner: QueryRunner,
32+
private readonly queryStorageDir: string,
33+
private readonly databaseItem: DatabaseItem,
34+
) {
1335
super(ctx);
1436
}
1537

@@ -49,5 +71,154 @@ export class DataExtensionsEditorView extends AbstractWebview<
4971

5072
protected async onWebViewLoaded() {
5173
super.onWebViewLoaded();
74+
75+
await this.loadExternalApiUsages();
76+
}
77+
78+
protected async loadExternalApiUsages(): Promise<void> {
79+
try {
80+
const queryResult = await this.runQuery();
81+
if (!queryResult) {
82+
await this.clearProgress();
83+
return;
84+
}
85+
86+
await this.showProgress({
87+
message: "Loading results",
88+
step: 1100,
89+
maxStep: 1500,
90+
});
91+
92+
const bqrsPath = queryResult.outputDir.bqrsPath;
93+
94+
const bqrsChunk = await this.getResults(bqrsPath);
95+
if (!bqrsChunk) {
96+
await this.clearProgress();
97+
return;
98+
}
99+
100+
await this.showProgress({
101+
message: "Finalizing results",
102+
step: 1450,
103+
maxStep: 1500,
104+
});
105+
106+
const externalApiUsages = decodeBqrsToExternalApiUsages(bqrsChunk);
107+
108+
await this.postMessage({
109+
t: "setExternalApiUsages",
110+
externalApiUsages,
111+
});
112+
113+
await this.clearProgress();
114+
} catch (err) {
115+
void showAndLogExceptionWithTelemetry(
116+
redactableError(
117+
asError(err),
118+
)`Failed to load external APi usages: ${getErrorMessage(err)}`,
119+
);
120+
}
121+
}
122+
123+
private async runQuery(): Promise<CoreCompletedQuery | undefined> {
124+
const qlpacks = await qlpackOfDatabase(this.cliServer, this.databaseItem);
125+
126+
const packsToSearch = [qlpacks.dbschemePack];
127+
if (qlpacks.queryPack) {
128+
packsToSearch.push(qlpacks.queryPack);
129+
}
130+
131+
const suiteFile = (
132+
await file({
133+
postfix: ".qls",
134+
})
135+
).path;
136+
const suiteYaml = [];
137+
for (const qlpack of packsToSearch) {
138+
suiteYaml.push({
139+
from: qlpack,
140+
queries: ".",
141+
include: {
142+
id: `${this.databaseItem.language}/telemetry/fetch-external-apis`,
143+
},
144+
});
145+
}
146+
await writeFile(suiteFile, dump(suiteYaml), "utf8");
147+
148+
const queries = await this.cliServer.resolveQueriesInSuite(
149+
suiteFile,
150+
getOnDiskWorkspaceFolders(),
151+
);
152+
153+
if (queries.length !== 1) {
154+
void extLogger.log(`Expected exactly one query, got ${queries.length}`);
155+
return;
156+
}
157+
158+
const query = queries[0];
159+
160+
const tokenSource = new CancellationTokenSource();
161+
162+
const queryRun = this.queryRunner.createQueryRun(
163+
this.databaseItem.databaseUri.fsPath,
164+
{ queryPath: query, quickEvalPosition: undefined },
165+
false,
166+
getOnDiskWorkspaceFolders(),
167+
undefined,
168+
this.queryStorageDir,
169+
undefined,
170+
undefined,
171+
);
172+
173+
return queryRun.evaluate(
174+
(update) => this.showProgress(update, 1500),
175+
tokenSource.token,
176+
new TeeLogger(this.queryRunner.logger, queryRun.outputDir.logPath),
177+
);
178+
}
179+
180+
private async getResults(bqrsPath: string) {
181+
const bqrsInfo = await this.cliServer.bqrsInfo(bqrsPath);
182+
if (bqrsInfo["result-sets"].length !== 1) {
183+
void extLogger.log(
184+
`Expected exactly one result set, got ${bqrsInfo["result-sets"].length}`,
185+
);
186+
return undefined;
187+
}
188+
189+
const resultSet = bqrsInfo["result-sets"][0];
190+
191+
await this.showProgress({
192+
message: "Decoding results",
193+
step: 1200,
194+
maxStep: 1500,
195+
});
196+
197+
return this.cliServer.bqrsDecode(bqrsPath, resultSet.name);
198+
}
199+
200+
/*
201+
* Progress in this class is a bit weird. Most of the progress is based on running the query.
202+
* Query progress is always between 0 and 1000. However, we still have some steps that need
203+
* to be done after the query has finished. Therefore, the maximum step is 1500. This captures
204+
* that there's 1000 steps of the query progress since that takes the most time, and then
205+
* an additional 500 steps for the rest of the work. The progress doesn't need to be 100%
206+
* accurate, so this is just a rough estimate.
207+
*/
208+
private async showProgress(update: ProgressUpdate, maxStep?: number) {
209+
await this.postMessage({
210+
t: "showProgress",
211+
step: update.step,
212+
maxStep: maxStep ?? update.maxStep,
213+
message: update.message,
214+
});
215+
}
216+
217+
private async clearProgress() {
218+
await this.showProgress({
219+
step: 0,
220+
maxStep: 0,
221+
message: "",
222+
});
52223
}
53224
}

‎extensions/ql-vscode/src/data-extensions-editor/external-api-usage.ts‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import { ResolvableLocationValue } from "../pure/bqrs-cli-types";
2+
3+
export type Call = {
4+
label: string;
5+
url: ResolvableLocationValue;
6+
};
7+
8+
export type ExternalApiUsage = {
9+
/**
10+
* Contains the full method signature, e.g. `org.sql2o.Connection#createQuery(String)`
11+
*/
12+
signature: string;
13+
packageName: string;
14+
typeName: string;
15+
methodName: string;
16+
methodParameters: string;
17+
supported: boolean;
18+
usages: Call[];
19+
};

‎extensions/ql-vscode/src/data-extensions-editor/modeled-method.ts‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
export type ModeledMethodType =
2+
| "none"
3+
| "source"
4+
| "sink"
5+
| "summary"
6+
| "neutral";
7+
8+
export type ModeledMethod = {
9+
type: ModeledMethodType;
10+
input: string;
11+
output: string;
12+
kind: string;
13+
};

‎extensions/ql-vscode/src/extension.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -865,7 +865,14 @@ async function activateWithInstalledDistribution(
865865
);
866866
ctx.subscriptions.push(localQueries);
867867

868-
const dataExtensionsEditorModule = new DataExtensionsEditorModule(ctx);
868+
const dataExtensionsEditorModule =
869+
await DataExtensionsEditorModule.initialize(
870+
ctx,
871+
dbm,
872+
cliServer,
873+
qs,
874+
tmpDir.name,
875+
);
869876

870877
void extLogger.log("Initializing QLTest interface.");
871878
const testExplorerExtension = extensions.getExtension<TestHub>(

0 commit comments

Comments
 (0)