From 1e818b6c2f3a3dd881ca54fa2b8ee8c614c382f9 Mon Sep 17 00:00:00 2001 From: Shane Ardell Date: Sat, 2 Nov 2019 13:35:47 -0500 Subject: [PATCH 1/4] remove duplicate timestampField input --- .../src/app/model/sensor-parser-config.ts | 4 +--- .../sensor-parser-config.component.html | 6 ------ .../sensor-parser-config.component.ts | 14 +++++++------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/metron-interface/metron-config/src/app/model/sensor-parser-config.ts b/metron-interface/metron-config/src/app/model/sensor-parser-config.ts index d83ef313bb..6b4da29e34 100644 --- a/metron-interface/metron-config/src/app/model/sensor-parser-config.ts +++ b/metron-interface/metron-config/src/app/model/sensor-parser-config.ts @@ -23,7 +23,7 @@ export class SensorParserConfig { writerClassName: string; errorWriterClassName: string; invalidWriterClassName: string; - parserConfig: {}; + parserConfig: { timestampField?: string }; fieldTransformations: FieldTransformer[]; numWorkers: number; numAckers: number; @@ -35,13 +35,11 @@ export class SensorParserConfig { errorWriterNumTasks: number; spoutConfig: {}; stormConfig: {}; - timestampField: string; constructor() { this.parserConfig = {}; this.fieldTransformations = []; this.spoutConfig = {}; this.stormConfig = {}; - this.timestampField = 'timestamp'; } } diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html index c85d37678a..5761ce1617 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html @@ -83,12 +83,6 @@ -
- - - -
-
diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts index 31ef9c0588..4877ffec70 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts @@ -78,6 +78,7 @@ export class SensorParserConfigComponent implements OnInit { grokStatement = ''; patternLabel = ''; currentSensors = []; + paramsID: string; editMode: boolean = false; @@ -191,8 +192,8 @@ export class SensorParserConfigComponent implements OnInit { ngOnInit() { this.route.params.subscribe(params => { - let id = params['id']; - this.init(id); + this.paramsID = params['id']; + this.init(this.paramsID); }); this.createForms(); this.getAvailableParsers(); @@ -210,10 +211,6 @@ export class SensorParserConfigComponent implements OnInit { this.sensorParserConfig.parserClassName, Validators.required ); - group['timestampField'] = new FormControl( - this.sensorParserConfig.timestampField, - Validators.required - ); group['grokStatement'] = new FormControl(this.grokStatement); group['transforms'] = new FormControl( this.sensorParserConfig['transforms'] @@ -317,6 +314,9 @@ export class SensorParserConfigComponent implements OnInit { } onParserTypeChange(): void { + if (this.paramsID === 'new' && this.isGrokParser(this.sensorParserConfig)) { + this.sensorParserConfig.parserConfig.timestampField = 'timestamp'; + } this.parserClassValid = this.sensorParserConfig.parserClassName !== undefined && this.sensorParserConfig.parserClassName.length > 0; @@ -338,7 +338,7 @@ export class SensorParserConfigComponent implements OnInit { this.kafkaTopicValid && this.parserClassValid && (!isGrokParser || this.isGrokStatementValid()) && - (!isGrokParser || !!this.sensorParserConfig.timestampField); + (!isGrokParser || !!this.sensorParserConfig.parserConfig.timestampField); } getKafkaStatus() { From 57688e9fbb9e776d3a1381cebb1a14c96266a581 Mon Sep 17 00:00:00 2001 From: Shane Ardell Date: Mon, 4 Nov 2019 11:51:54 -0600 Subject: [PATCH 2/4] fix parserConfig type --- .../metron-config/src/app/model/sensor-parser-config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metron-interface/metron-config/src/app/model/sensor-parser-config.ts b/metron-interface/metron-config/src/app/model/sensor-parser-config.ts index 6b4da29e34..96bdbbf0c6 100644 --- a/metron-interface/metron-config/src/app/model/sensor-parser-config.ts +++ b/metron-interface/metron-config/src/app/model/sensor-parser-config.ts @@ -23,7 +23,7 @@ export class SensorParserConfig { writerClassName: string; errorWriterClassName: string; invalidWriterClassName: string; - parserConfig: { timestampField?: string }; + parserConfig: { [key: string]: any; } fieldTransformations: FieldTransformer[]; numWorkers: number; numAckers: number; From 04d94aa1a58136869c77ea3e2da172b65bfa3904 Mon Sep 17 00:00:00 2001 From: Shane Ardell Date: Mon, 4 Nov 2019 11:52:14 -0600 Subject: [PATCH 3/4] update unit tests for parser config component --- .../sensor-parser-config.component.spec.ts | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts index 228125d3a8..e89c9e7e6e 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.spec.ts @@ -583,7 +583,7 @@ describe('Component: SensorParserConfig', () => { ); component.createForms(); - expect(Object.keys(component.sensorConfigForm.controls).length).toEqual(17); + expect(Object.keys(component.sensorConfigForm.controls).length).toEqual(16); expect( Object.keys(component.transformsValidationForm.controls).length ).toEqual(2); @@ -821,6 +821,7 @@ describe('Component: SensorParserConfig', () => { expect(component.isConfigValid()).toEqual(false); component.grokStatement = 'grok statement'; + component.sensorParserConfig.parserConfig.timestampField = 'timestamp'; expect(component.isConfigValid()).toEqual(true); })); @@ -1119,8 +1120,11 @@ describe('Component: SensorParserConfig', () => { expect(component.showAdvancedParserConfiguration).toEqual(false); })); - it('should be timestamp by default', () => { - expect(component.sensorParserConfig.timestampField).toEqual('timestamp'); + it('should be timestamp by default when parser is Grok', () => { + component.sensorParserConfig.parserClassName = 'org.apache.metron.parsers.GrokParser'; + component.paramsID = 'new'; + component.onParserTypeChange(); + expect(component.sensorParserConfig.parserConfig.timestampField).toEqual('timestamp'); }); it('should be invalid if timestamp field is empty', () => { @@ -1130,9 +1134,9 @@ describe('Component: SensorParserConfig', () => { component.grokStatement = 'grokStatement'; component.sensorParserConfig = new SensorParserConfig(); component.sensorParserConfig.parserClassName = 'org.apache.metron.parsers.GrokParser' - component.sensorParserConfig.timestampField = ''; + component.sensorParserConfig.parserConfig.timestampField = ''; expect(component.isConfigValid()).toEqual(false); - component.sensorParserConfig.timestampField = 'timestamp'; + component.sensorParserConfig.parserConfig.timestampField = 'timestamp'; expect(component.isConfigValid()).toEqual(true); }); }); From 8afa263b14b842eb6cc89a49f9a105038be718a3 Mon Sep 17 00:00:00 2001 From: Shane Ardell Date: Wed, 13 Nov 2019 22:35:45 -0500 Subject: [PATCH 4/4] add validation to timestampField input --- .../sensor-parser-config.component.html | 2 +- .../sensor-parser-config.component.ts | 2 ++ .../advanced-config-form.component.html | 15 ++++++++++++--- .../advanced-config-form.component.ts | 5 +++++ 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html index 5761ce1617..aefb05f427 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.html @@ -189,7 +189,7 @@
- +
diff --git a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts index 4877ffec70..ec1b163656 100644 --- a/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts +++ b/metron-interface/metron-config/src/app/sensors/sensor-parser-config/sensor-parser-config.component.ts @@ -316,6 +316,8 @@ export class SensorParserConfigComponent implements OnInit { onParserTypeChange(): void { if (this.paramsID === 'new' && this.isGrokParser(this.sensorParserConfig)) { this.sensorParserConfig.parserConfig.timestampField = 'timestamp'; + } else if (this.paramsID === 'new') { + delete this.sensorParserConfig.parserConfig.timestampField; } this.parserClassValid = this.sensorParserConfig.parserClassName !== undefined && diff --git a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html index 945be5fafa..960d78af85 100644 --- a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html +++ b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.html @@ -14,16 +14,25 @@
-
+
+ +
-
-
+
+ +
+
+
+ +
diff --git a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts index 5da9d48820..86386e9665 100644 --- a/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts +++ b/metron-interface/metron-config/src/app/shared/advanced-config-form/advanced-config-form.component.ts @@ -26,6 +26,7 @@ import {FormGroup, Validators, FormControl} from '@angular/forms'; export class AdvancedConfigFormComponent implements OnInit, OnChanges { @Input() config: {}; + @Input() parserClassName: string; configKeys: string[] = []; newConfigKey: string = 'enter field'; newConfigValue: string = 'enter value'; @@ -125,4 +126,8 @@ export class AdvancedConfigFormComponent implements OnInit, OnChanges { } + isGrokParser(): boolean { + return this.parserClassName === 'org.apache.metron.parsers.GrokParser'; + } + }