Skip to content

Commit dd49420

Browse files
taldekartaldekar
authored
feat: add configurable file indexing logic (#967)
* feat: add configurable file indexing logic * feat: add configurable file indexing logic * fix: update access for workspace index context * fix: add logic to read and respect user .gitignore rules * fix: add logic to read and respect user .gitignore rules * fix: add logic to read and respect user .gitignore rules * fix: upgrade to latest, correct runtimes package * fix: upgrade to latest, correct runtimes package * fix: update tests for chatDb * Revert "fix: upgrade to latest, correct runtimes package" This reverts commit 66f0fd4. * fix: conflict in package.json * fix: conflict in package.json * fix: conflict in package.json * Revert "fix: update tests for chatDb" This reverts commit a2ee818. * refactor: remove readline depenedncy and update index size upper bound * feat: add early termination logic when further filtering unnecessary * fix: remove gitignore-parser dependency * fix: configuration for max file and index size * refactor: use workspace/configuration call for all file indexing options * refactor: use workspace/configuration call for all file indexing options * fix: revert to original to prevent overflow issues * fix: parse LocalIndexConfig from workspace/configuration call * feat: add logging to file indexing logic * fix: add de-deplucation logic for init/dispose call * fix: agentic chat test bug * fix: configurationUtils test bug * fix: configurationUtils test bug * refactor: improve .gitignore parsing * fix: package-lock.json * fix: race condition in updateConfigurationHandler * fix: bug introduced by merge * feat: make index cache dir path configurable * feat: make index cache dir path configurable * fix: user homedir resolution * fix: user homedir resolution * fix: user homedir resolution * fix: ambiguity in file and index size parameter name * fix: revert optional logger change * fix: remove unused lock object * refactor: use import instead of require
1 parent 12dc8d7 commit dd49420

File tree

8 files changed

+279
-81
lines changed

8 files changed

+279
-81
lines changed

‎package-lock.json

Lines changed: 35 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎server/aws-lsp-codewhisperer/package.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,13 @@
4141
"deepmerge": "^4.3.1",
4242
"diff": "^7.0.0",
4343
"fastest-levenshtein": "^1.0.16",
44+
"fdir": "^6.4.3",
4445
"got": "^11.8.5",
4546
"hpagent": "^1.2.0",
47+
"ignore": "^7.0.3",
4648
"js-md5": "^0.8.3",
4749
"lokijs": "^1.5.12",
50+
"picomatch": "^4.0.2",
4851
"shlex": "2.1.2",
4952
"uuid": "^11.0.5",
5053
"vscode-uri": "^3.1.0"

‎server/aws-lsp-codewhisperer/src/language-server/agenticChat/tools/chatDb/util.test.ts

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -237,11 +237,15 @@ describe('ChatDb Utilities', () => {
237237

238238
it('should sort tabs by updatedAt in descending order within groups', () => {
239239
const now = new Date()
240-
const today1 = new Date(now)
241-
const today2 = new Date(now)
242-
today2.setHours(today2.getHours() - 1)
243-
const today3 = new Date(now)
244-
today3.setHours(today3.getHours() - 2)
240+
241+
const endOfDay = new Date(now)
242+
endOfDay.setHours(23, 59, 59, 999)
243+
244+
const today1 = new Date(endOfDay)
245+
const today2 = new Date(endOfDay)
246+
today2.setHours(endOfDay.getHours() - 1)
247+
const today3 = new Date(endOfDay)
248+
today3.setHours(endOfDay.getHours() - 2)
245249

246250
const tabs = [
247251
{

‎server/aws-lsp-codewhisperer/src/language-server/localProjectContext/localProjectContextServer.ts

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ export const LocalProjectContextServer = (): Server => features => {
1111
let localProjectContextController: LocalProjectContextController
1212
let amazonQServiceManager: AmazonQTokenServiceManager
1313
let telemetryService: TelemetryService
14+
let localProjectContextEnabled: boolean = false
1415

1516
lsp.addInitializer((params: InitializeParams) => {
1617
amazonQServiceManager = AmazonQTokenServiceManager.getInstance(features)
@@ -117,10 +118,18 @@ export const LocalProjectContextServer = (): Server => features => {
117118
const updateConfigurationHandler = async (updatedConfig: AmazonQWorkspaceConfig) => {
118119
logging.log('Updating configuration of local context server')
119120
try {
120-
logging.log(`Setting project context enabled to ${updatedConfig.projectContext?.enableLocalIndexing}`)
121-
updatedConfig.projectContext?.enableLocalIndexing
122-
? await localProjectContextController.init()
123-
: await localProjectContextController.dispose()
121+
if (localProjectContextEnabled !== updatedConfig.projectContext?.enableLocalIndexing) {
122+
localProjectContextEnabled = updatedConfig.projectContext?.enableLocalIndexing === true
123+
124+
logging.log(`Setting project context enabled to ${updatedConfig.projectContext?.enableLocalIndexing}`)
125+
localProjectContextEnabled
126+
? await localProjectContextController.init({
127+
ignoreFilePatterns: updatedConfig.projectContext?.localIndexing?.ignoreFilePatterns,
128+
maxFileSizeMB: updatedConfig.projectContext?.localIndexing?.maxFileSizeMB,
129+
maxIndexSizeMB: updatedConfig.projectContext?.localIndexing?.maxIndexSizeMB,
130+
})
131+
: await localProjectContextController.dispose()
132+
}
124133
} catch (error) {
125134
logging.error(`Error handling configuration change: ${error}`)
126135
}

‎server/aws-lsp-codewhisperer/src/shared/amazonQServiceManager/configurationUtils.test.ts

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,12 @@ describe('getAmazonQRelatedWorkspaceConfigs', () => {
1919
},
2020
projectContext: {
2121
enableLocalIndexing: true,
22+
localIndexing: {
23+
ignoreFilePatterns: [],
24+
maxFileSizeMB: 10,
25+
maxIndexSizeMB: 2048,
26+
indexCacheDirPath: undefined,
27+
},
2228
},
2329
}
2430

@@ -43,7 +49,10 @@ describe('getAmazonQRelatedWorkspaceConfigs', () => {
4349
inlineSuggestions: { extraContext: MOCKED_AWS_Q_SECTION.inlineSuggestions.extraContext },
4450
includeSuggestionsWithCodeReferences: MOCKED_AWS_CODEWHISPERER_SECTION.includeSuggestionsWithCodeReferences,
4551
shareCodeWhispererContentWithAWS: MOCKED_AWS_CODEWHISPERER_SECTION.shareCodeWhispererContentWithAWS,
46-
projectContext: { enableLocalIndexing: MOCKED_AWS_Q_SECTION.projectContext.enableLocalIndexing },
52+
projectContext: {
53+
enableLocalIndexing: MOCKED_AWS_Q_SECTION.projectContext.enableLocalIndexing,
54+
localIndexing: MOCKED_AWS_Q_SECTION.projectContext.localIndexing,
55+
},
4756
}
4857

4958
const amazonQConfig = await getAmazonQRelatedWorkspaceConfigs(features.lsp, features.logging)
@@ -87,6 +96,12 @@ describe('AmazonQConfigurationCache', () => {
8796
shareCodeWhispererContentWithAWS: true,
8897
projectContext: {
8998
enableLocalIndexing: true,
99+
localIndexing: {
100+
ignoreFilePatterns: [],
101+
maxFileSizeMB: 10,
102+
maxIndexSizeMB: 2048,
103+
indexCacheDirPath: undefined,
104+
},
90105
},
91106
}
92107
})

‎server/aws-lsp-codewhisperer/src/shared/amazonQServiceManager/configurationUtils.ts

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,16 @@ interface QInlineSuggestionsConfig {
6767
extraContext: string | undefined // aws.q.inlineSuggestions.extraContext
6868
}
6969

70+
interface LocalIndexConfig {
71+
ignoreFilePatterns?: string[]
72+
maxFileSizeMB?: number
73+
maxIndexSizeMB?: number
74+
indexCacheDirPath?: string
75+
}
76+
7077
interface QProjectContextConfig {
7178
enableLocalIndexing: boolean // aws.q.projectContext.enableLocalIndexing
79+
localIndexing?: LocalIndexConfig
7280
}
7381

7482
interface QConfigSection {
@@ -112,6 +120,12 @@ export async function getAmazonQRelatedWorkspaceConfigs(
112120
},
113121
projectContext: {
114122
enableLocalIndexing: newQConfig.projectContext?.enableLocalIndexing === true,
123+
localIndexing: {
124+
ignoreFilePatterns: newQConfig.projectContext?.localIndexing?.ignoreFilePatterns ?? [],
125+
maxFileSizeMB: newQConfig.projectContext?.localIndexing?.maxFileSizeMb ?? 10,
126+
maxIndexSizeMB: newQConfig.projectContext?.localIndexing?.maxIndexSizeMb ?? 2048,
127+
indexCacheDirPath: newQConfig.projectContext?.localIndexing?.indexCacheDirPath ?? undefined,
128+
},
115129
},
116130
}
117131

@@ -160,6 +174,12 @@ export const defaultAmazonQWorkspaceConfigFactory = (): AmazonQWorkspaceConfig =
160174
shareCodeWhispererContentWithAWS: false,
161175
projectContext: {
162176
enableLocalIndexing: false,
177+
localIndexing: {
178+
ignoreFilePatterns: [],
179+
maxFileSizeMB: 10,
180+
maxIndexSizeMB: 2048,
181+
indexCacheDirPath: undefined,
182+
},
163183
},
164184
}
165185
}

‎server/aws-lsp-codewhisperer/src/shared/localProjectContextController.test.ts

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ describe('LocalProjectContextController', () => {
6767

6868
describe('init', () => {
6969
it('should initialize vector library successfully', async () => {
70-
await controller.init(vectorLibMock)
70+
await controller.init({ vectorLib: vectorLibMock })
7171

7272
sinonAssert.notCalled(logging.error)
7373
sinonAssert.called(vectorLibMock.start)
@@ -78,15 +78,15 @@ describe('LocalProjectContextController', () => {
7878
it('should handle initialization errors', async () => {
7979
vectorLibMock.start.rejects(new Error('Init failed'))
8080

81-
await controller.init(vectorLibMock)
81+
await controller.init({ vectorLib: vectorLibMock })
8282

8383
sinonAssert.called(logging.error)
8484
})
8585
})
8686

8787
describe('queryVectorIndex', () => {
8888
beforeEach(async () => {
89-
await controller.init(vectorLibMock)
89+
await controller.init({ vectorLib: vectorLibMock })
9090
})
9191

9292
it('should return empty array when vector library is not initialized', async () => {
@@ -117,7 +117,7 @@ describe('LocalProjectContextController', () => {
117117

118118
describe('queryInlineProjectContext', () => {
119119
beforeEach(async () => {
120-
await controller.init(vectorLibMock)
120+
await controller.init({ vectorLib: vectorLibMock })
121121
})
122122

123123
it('should return empty array when vector library is not initialized', async () => {
@@ -160,7 +160,7 @@ describe('LocalProjectContextController', () => {
160160

161161
describe('updateIndex', () => {
162162
beforeEach(async () => {
163-
await controller.init(vectorLibMock)
163+
await controller.init({ vectorLib: vectorLibMock })
164164
})
165165

166166
it('should do nothing when vector library is not initialized', async () => {
@@ -191,7 +191,7 @@ describe('LocalProjectContextController', () => {
191191

192192
describe('dispose', () => {
193193
it('should clear and remove vector library reference', async () => {
194-
await controller.init(vectorLibMock)
194+
await controller.init({ vectorLib: vectorLibMock })
195195
await controller.dispose()
196196

197197
const vecLib = await vectorLibMock.start()

0 commit comments

Comments
 (0)