From 7dfe05622ea0fb7104f6bca25ad2e29ef43558ff Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 10:53:10 -0400 Subject: [PATCH 01/21] Add alignment checking for subqueries --- detection_rules/rule_validators.py | 10 ++++++++-- pyproject.toml | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 78cc0a348ec..34f8a23b042 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -929,11 +929,17 @@ def remote_validate_rule( # noqa: PLR0913 return response -def extract_error_field(source: str, exc: eql.EqlParseError | kql.KqlParseError) -> str | None: +def extract_error_field(source: str, exc: eql.EqlParseError | kql.KqlParseError, max_attempts: int = 10) -> str | None: """Extract the field name from an EQL or KQL parse error.""" lines = source.splitlines() mod = -1 if exc.line == len(lines) else 0 # type: ignore[reportUnknownMemberType] line = lines[exc.line + mod] # type: ignore[reportUnknownMemberType] - start = exc.column # type: ignore[reportUnknownMemberType] + start: int = exc.column # type: ignore[reportUnknownMemberType] + # Handle cases where subqueries cause column alignment to be off + for _ in range(max_attempts): + if line[start - 1] != " ": + start -= 1 + else: + break stop = start + len(exc.caret.strip()) # type: ignore[reportUnknownVariableType] return re.sub(r"^\W+|\W+$", "", line[start:stop]) # type: ignore[reportUnknownArgumentType] diff --git a/pyproject.toml b/pyproject.toml index c2808a4c731..79d3df01241 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "detection_rules" -version = "1.5.5" +version = "1.5.6" description = "Detection Rules is the home for rules used by Elastic Security. This repository is used for the development, maintenance, testing, validation, and release of rules for Elastic Security’s Detection Engine." readme = "README.md" requires-python = ">=3.12" From d382c068f55f31b4470db503dcd1a65d571ce4d1 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 10:53:56 -0400 Subject: [PATCH 02/21] add noqa --- detection_rules/rule_validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 34f8a23b042..13ac8d0fe13 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -938,7 +938,7 @@ def extract_error_field(source: str, exc: eql.EqlParseError | kql.KqlParseError, # Handle cases where subqueries cause column alignment to be off for _ in range(max_attempts): if line[start - 1] != " ": - start -= 1 + start -= 1 # type: ignore[reportUnknownMemberType] else: break stop = start + len(exc.caret.strip()) # type: ignore[reportUnknownVariableType] From 4b7e8b4022c19b4ee21c1128784aa8cca70d3e2b Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 11:03:54 -0400 Subject: [PATCH 03/21] Update to alpha num --- detection_rules/rule_validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 13ac8d0fe13..0ef25507458 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -937,8 +937,8 @@ def extract_error_field(source: str, exc: eql.EqlParseError | kql.KqlParseError, start: int = exc.column # type: ignore[reportUnknownMemberType] # Handle cases where subqueries cause column alignment to be off for _ in range(max_attempts): - if line[start - 1] != " ": - start -= 1 # type: ignore[reportUnknownMemberType] + if line[start - 1].isalnum() or line[start - 1] == ".": # type: ignore[reportUnknownMemberType] + start -= 1 # type: ignore[reportUnknownMemberType] else: break stop = start + len(exc.caret.strip()) # type: ignore[reportUnknownVariableType] From 2e1a738e66cee584cfc501d930e3ec6dfaeebc83 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 11:47:15 -0400 Subject: [PATCH 04/21] Add additional format check --- detection_rules/rule_validators.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 0ef25507458..65a2e921b5c 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -931,7 +931,10 @@ def remote_validate_rule( # noqa: PLR0913 def extract_error_field(source: str, exc: eql.EqlParseError | kql.KqlParseError, max_attempts: int = 10) -> str | None: """Extract the field name from an EQL or KQL parse error.""" - lines = source.splitlines() + # If error reported in subquery, adjust source accordingly + if exc.source != source: # type: ignore[reportUnknownMemberType] + source = exc.source # type: ignore[reportUnknownMemberType] + lines = source.splitlines() # type: ignore[reportUnknownMemberType] mod = -1 if exc.line == len(lines) else 0 # type: ignore[reportUnknownMemberType] line = lines[exc.line + mod] # type: ignore[reportUnknownMemberType] start: int = exc.column # type: ignore[reportUnknownMemberType] From 434a0cb8528232e7f7bcb1b2010f38b30cc10b16 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 13:20:20 -0400 Subject: [PATCH 05/21] Update to handle multiple exc source formats --- detection_rules/rule_validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 65a2e921b5c..2bce71aa7e4 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -931,8 +931,8 @@ def remote_validate_rule( # noqa: PLR0913 def extract_error_field(source: str, exc: eql.EqlParseError | kql.KqlParseError, max_attempts: int = 10) -> str | None: """Extract the field name from an EQL or KQL parse error.""" - # If error reported in subquery, adjust source accordingly - if exc.source != source: # type: ignore[reportUnknownMemberType] + # If error reported in subquery and exc references exc.source rather than source, adjust source accordingly + if exc.source != source and len(exc.source.splitlines()) > exc.line: # type: ignore[reportUnknownMemberType] source = exc.source # type: ignore[reportUnknownMemberType] lines = source.splitlines() # type: ignore[reportUnknownMemberType] mod = -1 if exc.line == len(lines) else 0 # type: ignore[reportUnknownMemberType] From f4b50880df080fe11d560223c1ad308f18e875cc Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 13:39:06 -0400 Subject: [PATCH 06/21] Allow field to be over written with original field --- detection_rules/rule_validators.py | 36 +++++++++++++----------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 2bce71aa7e4..9a20d590feb 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -373,9 +373,13 @@ def text_fields(self, eql_schema: ecs.KqlSchema2Eql | endgame.EndgameSchema) -> def unique_fields(self) -> list[str]: # type: ignore[reportIncompatibleMethodOverride] return list({str(f) for f in self.ast if isinstance(f, eql.ast.Field)}) # type: ignore[reportUnknownVariableType] - def auto_add_field(self, validation_checks_error: eql.EqlParseError, index_or_dataview: str) -> None: + def auto_add_field( + self, validation_checks_error: eql.EqlParseError, index_or_dataview: str, field: str | None = None + ) -> None: """Auto add a missing field to the schema.""" - field_name = extract_error_field(self.query, validation_checks_error) + field_name = field + if not field: + field_name = extract_error_field(self.query, validation_checks_error) if not field_name: raise ValueError("No field name found") field_type = ecs.get_all_flattened_schema().get(field_name) @@ -606,7 +610,7 @@ def validate(self, data: "QueryRuleData", meta: RuleMeta, max_attempts: int = 10 ) first_error: EQL_ERROR_TYPES | ValueError | None = None for t in ordered_targets: - exc = self.validate_query_text_with_schema( + exc, field = self.validate_query_text_with_schema( t.query_text, t.schema, err_trailer=t.err_trailer, @@ -629,7 +633,7 @@ def validate(self, data: "QueryRuleData", meta: RuleMeta, max_attempts: int = 10 and RULES_CONFIG.auto_gen_schema_file and data.index_or_dataview ): - self.auto_add_field(first_error, data.index_or_dataview[0]) # type: ignore[reportArgumentType] + self.auto_add_field(first_error, data.index_or_dataview[0], field=field) # type: ignore[reportArgumentType] continue # Raise the enriched parse error (includes target trailer + metadata) @@ -645,7 +649,7 @@ def validate_query_text_with_schema( # noqa: PLR0913 min_stack_version: str, beat_types: list[str] | None = None, integration_types: list[str] | None = None, - ) -> EQL_ERROR_TYPES | ValueError | None: + ) -> tuple[EQL_ERROR_TYPES | ValueError | None, str | None]: """Validate the provided EQL query text against the schema (variant of validate_query_with_schema).""" try: config = set_eql_config(min_stack_version) @@ -663,7 +667,7 @@ def validate_query_text_with_schema( # noqa: PLR0913 and ("Unknown field" in message or "Field not recognized" in message) and f"?{field}" in self.query ): - return None + return None, field if "Unknown field" in message and beat_types: trailer_parts.insert(0, "Try adding event.module or event.dataset to specify beats module") elif "Field not recognized" in message and isinstance(schema, ecs.KqlSchema2Eql): @@ -691,10 +695,11 @@ def validate_query_text_with_schema( # noqa: PLR0913 exc.source, # type: ignore[reportUnknownArgumentType] len(exc.caret.lstrip()), trailer=trailer, - ) + ), field except Exception as exc: # noqa: BLE001 print(err_trailer) - return exc # type: ignore[reportReturnType] + return exc, None # type: ignore[reportReturnType] + return None, None def validate_rule_type_configurations(self, data: EQLRuleData, meta: RuleMeta) -> tuple[list[str], bool]: """Validate EQL rule type configurations (timestamp_field, event_category_override, tiebreaker_field). @@ -929,20 +934,11 @@ def remote_validate_rule( # noqa: PLR0913 return response -def extract_error_field(source: str, exc: eql.EqlParseError | kql.KqlParseError, max_attempts: int = 10) -> str | None: +def extract_error_field(source: str, exc: eql.EqlParseError | kql.KqlParseError) -> str | None: """Extract the field name from an EQL or KQL parse error.""" - # If error reported in subquery and exc references exc.source rather than source, adjust source accordingly - if exc.source != source and len(exc.source.splitlines()) > exc.line: # type: ignore[reportUnknownMemberType] - source = exc.source # type: ignore[reportUnknownMemberType] - lines = source.splitlines() # type: ignore[reportUnknownMemberType] + lines = source.splitlines() mod = -1 if exc.line == len(lines) else 0 # type: ignore[reportUnknownMemberType] line = lines[exc.line + mod] # type: ignore[reportUnknownMemberType] - start: int = exc.column # type: ignore[reportUnknownMemberType] - # Handle cases where subqueries cause column alignment to be off - for _ in range(max_attempts): - if line[start - 1].isalnum() or line[start - 1] == ".": # type: ignore[reportUnknownMemberType] - start -= 1 # type: ignore[reportUnknownMemberType] - else: - break + start = exc.column # type: ignore[reportUnknownMemberType] stop = start + len(exc.caret.strip()) # type: ignore[reportUnknownVariableType] return re.sub(r"^\W+|\W+$", "", line[start:stop]) # type: ignore[reportUnknownArgumentType] From 594f58fc1c954fecb372053e6f3976e0084119d8 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 13:42:39 -0400 Subject: [PATCH 07/21] Add comment explaining change --- detection_rules/rule_validators.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index 9a20d590feb..e3a70d18bf8 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -661,6 +661,9 @@ def validate_query_text_with_schema( # noqa: PLR0913 # If the error is an unknown field and the field was referenced as optional (prefixed with '?'), # treat this target as non-fatal to honor EQL optional semantics. + # To support EQL sequence and sub query validation we need to return this field to overwrite + # what would have been parsed via auto_add_field as the error message and query may be our of sync + # depending on how the method is called. field = extract_error_field(query_text, exc) if ( field From 373877ca958de6efea0723310a4b11166decb1a3 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 16:04:32 -0400 Subject: [PATCH 08/21] Adjusted to better match Kibana --- detection_rules/schemas/definitions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/schemas/definitions.py b/detection_rules/schemas/definitions.py index 0fd2be2e4ed..762e9786b18 100644 --- a/detection_rules/schemas/definitions.py +++ b/detection_rules/schemas/definitions.py @@ -245,7 +245,7 @@ def validator_wrapper(value: Any) -> Any: list[NonEmptyStr], fields.List(NON_EMPTY_STRING_FIELD, validate=validate.Length(min=1, max=3)) ] PositiveInteger = Annotated[int, fields.Integer(validate=validate.Range(min=1))] -RiskScore = Annotated[int, fields.Integer(validate=validate.Range(min=1, max=100))] +RiskScore = Annotated[int, fields.Integer(validate=validate.Range(min=0, max=100))] RuleName = Annotated[str, fields.String(validate=elastic_rule_name_regexp(NAME_PATTERN))] SemVer = Annotated[str, fields.String(validate=validate.Regexp(VERSION_PATTERN))] SemVerMinorOnly = Annotated[str, fields.String(validate=validate.Regexp(MINOR_SEMVER))] From 69eedd07dbcbcf953d7eb483d60e0969b49da8b1 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 16:30:21 -0400 Subject: [PATCH 09/21] Fix rule IDs now that we have schema enforcement --- detection_rules/etc/custom-consolidated-rules.ndjson | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/detection_rules/etc/custom-consolidated-rules.ndjson b/detection_rules/etc/custom-consolidated-rules.ndjson index 7a295c9cf74..a7c9f279f06 100644 --- a/detection_rules/etc/custom-consolidated-rules.ndjson +++ b/detection_rules/etc/custom-consolidated-rules.ndjson @@ -1,12 +1,12 @@ -{"id":"49954888-3d9a-44fd-b224-8f8e9653d294","updated_at":"2025-08-18T03:39:54.977Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.318Z","created_by":"841510929","name":"test_kql_rule","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":50,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"process_started_by_ms_office_program","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[{"package":"o365","version":"^2.3.2"}],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"meta":{"type":"phrase","key":"event.action","params":{"query":"Process Create (rule: ProcessCreate)"},"disabled":false,"negate":false},"$state":{"store":"appState"},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"actions":[]} +{"id":"49954888-3d9a-44fd-b224-8f8e9653d294","updated_at":"2025-08-18T03:39:54.977Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.318Z","created_by":"841510929","name":"test_kql_rule","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":50,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"bcbd5906-fc38-4cbe-8b54-c2dba5d4b127","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[{"package":"o365","version":"^2.3.2"}],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"meta":{"type":"phrase","key":"event.action","params":{"query":"Process Create (rule: ProcessCreate)"},"disabled":false,"negate":false},"$state":{"store":"appState"},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"actions":[]} {"id":"c7c868c0-cfe1-4139-a873-4c8ce7b181c1","updated_at":"2025-08-18T03:41:10.096Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.310Z","created_by":"841510929","name":"test_kql_with_alert_supprestion_and_investigation_fileds","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":50,"severity":"low","note":"This a a test sample investigation Guide\nThis a a test sample investigation Guide\nThis a a test sample investigation Guide\n\n!{osquery{\"query\":\"SELECT * FROM file WHERE ( path LIKE '/etc/ld.so.conf.d/%' OR path LIKE '/etc/cron.d/%' OR path LIKE '/etc/sudoers.d/%'\\nOR path LIKE '/etc/rc%.d/%' OR path LIKE '/etc/init.d/%' OR path LIKE '/etc/systemd/system/%' OR path LIKE\\n'/usr/lib/systemd/system/%' )\",\"label\":\"test-osquery\"}}\n\n!{investigate{\"label\":\"test-investigation-query\",\"description\":\"test-investigation-query\",\"providers\":[[{\"field\":\"host.name\",\"excluded\":false,\"queryType\":\"phrase\",\"value\":\"test-host\",\"valueType\":\"string\"}]]}}","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"742feb36-ac4c-45e0-b8a5-3b3cfa66b6d2","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"$state":{"store":"appState"},"meta":{"disabled":false,"key":"event.action","negate":false,"type":"phrase","params":{"query":"Process Create (rule: ProcessCreate)"}},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"alert_suppression":{"group_by":["process.parent.name"],"duration":{"value":5,"unit":"h"},"missing_fields_strategy":"suppress"},"actions":[]} -{"id":"e9430a4c-5fce-41b7-9d55-7645360e11d9","updated_at":"2025-08-18T03:40:30.081Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.326Z","created_by":"841510929","name":"test_kql_with_alert_suppression","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":50,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"process_started_by_ms_office_program_supression","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"meta":{"type":"phrase","key":"event.action","params":{"query":"Process Create (rule: ProcessCreate)"},"disabled":false,"negate":false},"$state":{"store":"appState"},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"alert_suppression":{"group_by":["process.parent.name"],"duration":{"value":5,"unit":"h"},"missing_fields_strategy":"suppress"},"actions":[]} -{"id":"45241dcf-1bb2-41eb-8e91-89741af275c0","updated_at":"2025-08-18T03:43:41.240Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.317Z","created_by":"841510929","name":"test_eql_rule","tags":["EQL","Windows","rundll32.exe"],"interval":"5m","enabled":true,"revision":1,"description":"Unusual rundll32.exe network connection","risk_score":21,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"eql-outbound-rundll32-connections","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"event.type","type":"keyword","ecs":true},{"name":"process.args","type":"keyword","ecs":true},{"name":"process.args_count","type":"long","ecs":true},{"name":"process.entity_id","type":"keyword","ecs":true},{"name":"process.name","type":"keyword","ecs":true},{"name":"process.pe.original_file_name","type":"keyword","ecs":true}],"setup":"None","type":"eql","language":"eql","index":["logs-*"],"query":"sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n","filters":[],"actions":[]} +{"id":"e9430a4c-5fce-41b7-9d55-7645360e11d9","updated_at":"2025-08-18T03:40:30.081Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.326Z","created_by":"841510929","name":"test_kql_with_alert_suppression","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":50,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"2c6c5352-11cb-40a5-9294-e61ef5f1954f","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"meta":{"type":"phrase","key":"event.action","params":{"query":"Process Create (rule: ProcessCreate)"},"disabled":false,"negate":false},"$state":{"store":"appState"},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"alert_suppression":{"group_by":["process.parent.name"],"duration":{"value":5,"unit":"h"},"missing_fields_strategy":"suppress"},"actions":[]} +{"id":"45241dcf-1bb2-41eb-8e91-89741af275c0","updated_at":"2025-08-18T03:43:41.240Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.317Z","created_by":"841510929","name":"test_eql_rule","tags":["EQL","Windows","rundll32.exe"],"interval":"5m","enabled":true,"revision":1,"description":"Unusual rundll32.exe network connection","risk_score":21,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2cc8f325-e1b1-4201-8b8d-88a51c94992b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"event.type","type":"keyword","ecs":true},{"name":"process.args","type":"keyword","ecs":true},{"name":"process.args_count","type":"long","ecs":true},{"name":"process.entity_id","type":"keyword","ecs":true},{"name":"process.name","type":"keyword","ecs":true},{"name":"process.pe.original_file_name","type":"keyword","ecs":true}],"setup":"None","type":"eql","language":"eql","index":["logs-*"],"query":"sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n","filters":[],"actions":[]} {"id":"11d7b970-0076-4ae1-b328-16d6778489f2","updated_at":"2025-08-18T03:45:34.509Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.308Z","created_by":"841510929","name":"test_esql_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Find Excel events","risk_score":21,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"7e0f6dae-5847-465f-89e9-a6de0e9ef918","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"esql","language":"esql","query":"from auditbeat-8.10.2 METADATA _id, _version, _index | KEEP process.parent.name | where process.parent.name == \"EXCEL.EXE\"\n","actions":[]} {"id":"72abd101-fe39-43f0-a6d1-e9a373684cab","updated_at":"2025-08-18T03:46:00.515Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.334Z","created_by":"841510929","name":"test_new_terms_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Detects a user associated with a new IP address","risk_score":21,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2390c9dd-ad90-4af6-97a4-1d607ba0f092","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"user.id","type":"keyword","ecs":true},{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"new_terms","query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","new_terms_fields":["user.id","source.ip"],"history_window_start":"now-30d","index":["auditbeat*"],"filters":[],"language":"kuery","actions":[]} {"id":"e0e31a34-2e18-40c0-af09-539021e8439d","updated_at":"2025-08-18T03:47:21.590Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.344Z","created_by":"841510929","name":"test_indicator_match_rule_with_email_actions","tags":[],"interval":"5m","enabled":true,"revision":5,"description":"Checks for bad IP addresses listed in the ip-threat-list index","risk_score":50,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator","actions":[{"id":"elastic-cloud-email","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","subject":"Test Actions","to":["tradebot-elastic@elastic.com"]},"action_type_id":".email","uuid":"74c388a4-c94f-4541-bacc-2a1b4c47e768","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} -{"id":"a0d623ea-e8a4-4eff-9c6c-643ceff9f3e5","updated_at":"2025-08-18T03:44:54.407Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.331Z","created_by":"841510929","name":"test_threshold_with_rule_exception","tags":["Brute force"],"interval":"2m","enabled":true,"revision":1,"description":"Detects when there are 20 or more failed login attempts from the same IP address with a 2 minute time frame.","risk_score":30,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-3m","rule_id":"liv-win-ser-logins","max_signals":100,"risk_score_mapping":[],"severity_mapping":[{"field":"source.geo.city_name","operator":"equals","severity":"low","value":"Manchester"},{"field":"source.geo.city_name","operator":"equals","severity":"medium","value":"London"},{"field":"source.geo.city_name","operator":"equals","severity":"high","value":"Birmingham"},{"field":"source.geo.city_name","operator":"equals","severity":"critical","value":"Wallingford"}],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"82395156-8ad2-46c3-be79-1f1a23c0d802","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","type":"rule_default","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"threshold","language":"kuery","index":["winlogbeat-*"],"query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","filters":[],"threshold":{"field":["source.ip"],"value":20,"cardinality":[]},"actions":[]} -{"id":"9bcffa42-d8b5-4706-afec-3cf33b19d9b1","updated_at":"2025-08-18T03:48:19.634Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.415Z","created_by":"841510929","name":"test_machine_learning_rule_with_index_action_connector ","tags":["machine learning","Linux"],"interval":"5m","enabled":true,"revision":5,"description":"Generates alerts when the job discovers anomalies over 70","risk_score":70,"severity":"high","note":"Shut down the internet.","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"ml_linux_network_high_threshold","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[],"setup":"This rule requires data coming in from Elastic Defend.","type":"machine_learning","anomaly_threshold":70,"machine_learning_job_id":["linux_anomalous_network_activity_ecs"],"actions":[{"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","params":{"documents":[{"rule.id":"{{rule.id}}"}]},"action_type_id":".index","uuid":"175f50f8-3bc1-4017-805f-e532d7eb2f91","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} +{"id":"a0d623ea-e8a4-4eff-9c6c-643ceff9f3e5","updated_at":"2025-08-18T03:44:54.407Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.331Z","created_by":"841510929","name":"test_threshold_with_rule_exception","tags":["Brute force"],"interval":"2m","enabled":true,"revision":1,"description":"Detects when there are 20 or more failed login attempts from the same IP address with a 2 minute time frame.","risk_score":30,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-3m","rule_id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[{"field":"source.geo.city_name","operator":"equals","severity":"low","value":"Manchester"},{"field":"source.geo.city_name","operator":"equals","severity":"medium","value":"London"},{"field":"source.geo.city_name","operator":"equals","severity":"high","value":"Birmingham"},{"field":"source.geo.city_name","operator":"equals","severity":"critical","value":"Wallingford"}],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"82395156-8ad2-46c3-be79-1f1a23c0d802","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","type":"rule_default","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"threshold","language":"kuery","index":["winlogbeat-*"],"query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","filters":[],"threshold":{"field":["source.ip"],"value":20,"cardinality":[]},"actions":[]} +{"id":"9bcffa42-d8b5-4706-afec-3cf33b19d9b1","updated_at":"2025-08-18T03:48:19.634Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.415Z","created_by":"841510929","name":"test_machine_learning_rule_with_index_action_connector ","tags":["machine learning","Linux"],"interval":"5m","enabled":true,"revision":5,"description":"Generates alerts when the job discovers anomalies over 70","risk_score":70,"severity":"high","note":"Shut down the internet.","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[],"setup":"This rule requires data coming in from Elastic Defend.","type":"machine_learning","anomaly_threshold":70,"machine_learning_job_id":["linux_anomalous_network_activity_ecs"],"actions":[{"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","params":{"documents":[{"rule.id":"{{rule.id}}"}]},"action_type_id":".index","uuid":"175f50f8-3bc1-4017-805f-e532d7eb2f91","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} {"_version":"WzE3NjU1LDhd","created_at":"2025-08-14T12:42:04.522Z","created_by":"841510929","description":"","id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","immutable":false,"list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","name":"Test Excpetion List","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"14b3565d-0c8a-48db-b76a-e46c01574a57","type":"detection","updated_at":"2025-08-14T12:42:04.522Z","updated_by":"841510929","version":1} {"_version":"WzE3NjU2LDhd","comments":[],"created_at":"2025-08-14T12:42:34.361Z","created_by":"841510929","description":"Exception list item","entries":[{"type":"match","field":"host.name","value":"test-host","operator":"included"}],"id":"dc084b23-4b9c-40c9-a172-77468ee2a4d9","item_id":"734852b6-b3bf-4942-8b3b-c058bd16088f","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","name":"host_excpetion","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"50c46edf-691b-4397-ad9e-e06a544a81d0","type":"simple","updated_at":"2025-08-14T12:42:34.361Z","updated_by":"841510929"} {"_version":"WzE3NjUwLDhd","created_at":"2025-08-14T12:19:29.454Z","created_by":"841510929","description":"Exception list containing exceptions for rule with id: 51a51212-5975-45ac-b909-c7840a903141","id":"82395156-8ad2-46c3-be79-1f1a23c0d802","immutable":false,"list_id":"0a4124f8-2074-450b-8689-d7dee319c666","name":"Exceptions for rule - Test Windows server prml-19","namespace_type":"single","os_types":[],"tags":["default_rule_exception_list"],"tie_breaker_id":"46a0d0b5-8793-4f60-a20b-6f76274b1722","type":"rule_default","updated_at":"2025-08-14T12:19:29.454Z","updated_by":"841510929","version":1} From 9aedd6d4ca8b1095a48eb88244c12ca33029ec22 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 28 Oct 2025 16:53:47 -0400 Subject: [PATCH 10/21] Update rule prompt to allow for int 0 values --- detection_rules/cli_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/detection_rules/cli_utils.py b/detection_rules/cli_utils.py index 27201bd63f9..2117fccec30 100644 --- a/detection_rules/cli_utils.py +++ b/detection_rules/cli_utils.py @@ -92,11 +92,11 @@ def _convert_type(_val: Any) -> Any: ) while True: - result = value or input(prompt) or default + result = value if value is not None else input(prompt) or default if result == "n/a": result = None - if not result: + if result is None: if is_required: value = None continue @@ -318,7 +318,7 @@ def rule_prompt( # noqa: PLR0912, PLR0913, PLR0915 contents[name] = threat_map continue - if kwargs.get(name): + if name in kwargs: contents[name] = schema_prompt(name, value=kwargs.pop(name)) continue From 041b41883542699a939e4e765e8b6dcc5099c1a6 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 29 Oct 2025 10:29:53 -0400 Subject: [PATCH 11/21] Update Audit Beat --- .../etc/custom-consolidated-rules.ndjson | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/detection_rules/etc/custom-consolidated-rules.ndjson b/detection_rules/etc/custom-consolidated-rules.ndjson index a7c9f279f06..c6c969e0eb8 100644 --- a/detection_rules/etc/custom-consolidated-rules.ndjson +++ b/detection_rules/etc/custom-consolidated-rules.ndjson @@ -1,12 +1,12 @@ -{"id":"49954888-3d9a-44fd-b224-8f8e9653d294","updated_at":"2025-08-18T03:39:54.977Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.318Z","created_by":"841510929","name":"test_kql_rule","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":50,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"bcbd5906-fc38-4cbe-8b54-c2dba5d4b127","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[{"package":"o365","version":"^2.3.2"}],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"meta":{"type":"phrase","key":"event.action","params":{"query":"Process Create (rule: ProcessCreate)"},"disabled":false,"negate":false},"$state":{"store":"appState"},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"actions":[]} -{"id":"c7c868c0-cfe1-4139-a873-4c8ce7b181c1","updated_at":"2025-08-18T03:41:10.096Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.310Z","created_by":"841510929","name":"test_kql_with_alert_supprestion_and_investigation_fileds","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":50,"severity":"low","note":"This a a test sample investigation Guide\nThis a a test sample investigation Guide\nThis a a test sample investigation Guide\n\n!{osquery{\"query\":\"SELECT * FROM file WHERE ( path LIKE '/etc/ld.so.conf.d/%' OR path LIKE '/etc/cron.d/%' OR path LIKE '/etc/sudoers.d/%'\\nOR path LIKE '/etc/rc%.d/%' OR path LIKE '/etc/init.d/%' OR path LIKE '/etc/systemd/system/%' OR path LIKE\\n'/usr/lib/systemd/system/%' )\",\"label\":\"test-osquery\"}}\n\n!{investigate{\"label\":\"test-investigation-query\",\"description\":\"test-investigation-query\",\"providers\":[[{\"field\":\"host.name\",\"excluded\":false,\"queryType\":\"phrase\",\"value\":\"test-host\",\"valueType\":\"string\"}]]}}","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"742feb36-ac4c-45e0-b8a5-3b3cfa66b6d2","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"$state":{"store":"appState"},"meta":{"disabled":false,"key":"event.action","negate":false,"type":"phrase","params":{"query":"Process Create (rule: ProcessCreate)"}},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"alert_suppression":{"group_by":["process.parent.name"],"duration":{"value":5,"unit":"h"},"missing_fields_strategy":"suppress"},"actions":[]} -{"id":"e9430a4c-5fce-41b7-9d55-7645360e11d9","updated_at":"2025-08-18T03:40:30.081Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.326Z","created_by":"841510929","name":"test_kql_with_alert_suppression","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":50,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"2c6c5352-11cb-40a5-9294-e61ef5f1954f","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"meta":{"type":"phrase","key":"event.action","params":{"query":"Process Create (rule: ProcessCreate)"},"disabled":false,"negate":false},"$state":{"store":"appState"},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"alert_suppression":{"group_by":["process.parent.name"],"duration":{"value":5,"unit":"h"},"missing_fields_strategy":"suppress"},"actions":[]} -{"id":"45241dcf-1bb2-41eb-8e91-89741af275c0","updated_at":"2025-08-18T03:43:41.240Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.317Z","created_by":"841510929","name":"test_eql_rule","tags":["EQL","Windows","rundll32.exe"],"interval":"5m","enabled":true,"revision":1,"description":"Unusual rundll32.exe network connection","risk_score":21,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2cc8f325-e1b1-4201-8b8d-88a51c94992b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"event.type","type":"keyword","ecs":true},{"name":"process.args","type":"keyword","ecs":true},{"name":"process.args_count","type":"long","ecs":true},{"name":"process.entity_id","type":"keyword","ecs":true},{"name":"process.name","type":"keyword","ecs":true},{"name":"process.pe.original_file_name","type":"keyword","ecs":true}],"setup":"None","type":"eql","language":"eql","index":["logs-*"],"query":"sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n","filters":[],"actions":[]} -{"id":"11d7b970-0076-4ae1-b328-16d6778489f2","updated_at":"2025-08-18T03:45:34.509Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.308Z","created_by":"841510929","name":"test_esql_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Find Excel events","risk_score":21,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"7e0f6dae-5847-465f-89e9-a6de0e9ef918","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"esql","language":"esql","query":"from auditbeat-8.10.2 METADATA _id, _version, _index | KEEP process.parent.name | where process.parent.name == \"EXCEL.EXE\"\n","actions":[]} -{"id":"72abd101-fe39-43f0-a6d1-e9a373684cab","updated_at":"2025-08-18T03:46:00.515Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.334Z","created_by":"841510929","name":"test_new_terms_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Detects a user associated with a new IP address","risk_score":21,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2390c9dd-ad90-4af6-97a4-1d607ba0f092","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"user.id","type":"keyword","ecs":true},{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"new_terms","query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","new_terms_fields":["user.id","source.ip"],"history_window_start":"now-30d","index":["auditbeat*"],"filters":[],"language":"kuery","actions":[]} -{"id":"e0e31a34-2e18-40c0-af09-539021e8439d","updated_at":"2025-08-18T03:47:21.590Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.344Z","created_by":"841510929","name":"test_indicator_match_rule_with_email_actions","tags":[],"interval":"5m","enabled":true,"revision":5,"description":"Checks for bad IP addresses listed in the ip-threat-list index","risk_score":50,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator","actions":[{"id":"elastic-cloud-email","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","subject":"Test Actions","to":["tradebot-elastic@elastic.com"]},"action_type_id":".email","uuid":"74c388a4-c94f-4541-bacc-2a1b4c47e768","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} -{"id":"a0d623ea-e8a4-4eff-9c6c-643ceff9f3e5","updated_at":"2025-08-18T03:44:54.407Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.331Z","created_by":"841510929","name":"test_threshold_with_rule_exception","tags":["Brute force"],"interval":"2m","enabled":true,"revision":1,"description":"Detects when there are 20 or more failed login attempts from the same IP address with a 2 minute time frame.","risk_score":30,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-3m","rule_id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[{"field":"source.geo.city_name","operator":"equals","severity":"low","value":"Manchester"},{"field":"source.geo.city_name","operator":"equals","severity":"medium","value":"London"},{"field":"source.geo.city_name","operator":"equals","severity":"high","value":"Birmingham"},{"field":"source.geo.city_name","operator":"equals","severity":"critical","value":"Wallingford"}],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"82395156-8ad2-46c3-be79-1f1a23c0d802","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","type":"rule_default","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"threshold","language":"kuery","index":["winlogbeat-*"],"query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","filters":[],"threshold":{"field":["source.ip"],"value":20,"cardinality":[]},"actions":[]} -{"id":"9bcffa42-d8b5-4706-afec-3cf33b19d9b1","updated_at":"2025-08-18T03:48:19.634Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.415Z","created_by":"841510929","name":"test_machine_learning_rule_with_index_action_connector ","tags":["machine learning","Linux"],"interval":"5m","enabled":true,"revision":5,"description":"Generates alerts when the job discovers anomalies over 70","risk_score":70,"severity":"high","note":"Shut down the internet.","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[],"setup":"This rule requires data coming in from Elastic Defend.","type":"machine_learning","anomaly_threshold":70,"machine_learning_job_id":["linux_anomalous_network_activity_ecs"],"actions":[{"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","params":{"documents":[{"rule.id":"{{rule.id}}"}]},"action_type_id":".index","uuid":"175f50f8-3bc1-4017-805f-e532d7eb2f91","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} +{"id":"49954888-3d9a-44fd-b224-8f8e9653d294","updated_at":"2025-08-18T03:39:54.977Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.318Z","created_by":"841510929","name":"test_kql_rule","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"bcbd5906-fc38-4cbe-8b54-c2dba5d4b127","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[{"package":"o365","version":"^2.3.2"}],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"meta":{"type":"phrase","key":"event.action","params":{"query":"Process Create (rule: ProcessCreate)"},"disabled":false,"negate":false},"$state":{"store":"appState"},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"actions":[]} +{"id":"c7c868c0-cfe1-4139-a873-4c8ce7b181c1","updated_at":"2025-08-18T03:41:10.096Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.310Z","created_by":"841510929","name":"test_kql_with_alert_supprestion_and_investigation_fileds","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":0,"severity":"low","note":"This a a test sample investigation Guide\nThis a a test sample investigation Guide\nThis a a test sample investigation Guide\n\n!{osquery{\"query\":\"SELECT * FROM file WHERE ( path LIKE '/etc/ld.so.conf.d/%' OR path LIKE '/etc/cron.d/%' OR path LIKE '/etc/sudoers.d/%'\\nOR path LIKE '/etc/rc%.d/%' OR path LIKE '/etc/init.d/%' OR path LIKE '/etc/systemd/system/%' OR path LIKE\\n'/usr/lib/systemd/system/%' )\",\"label\":\"test-osquery\"}}\n\n!{investigate{\"label\":\"test-investigation-query\",\"description\":\"test-investigation-query\",\"providers\":[[{\"field\":\"host.name\",\"excluded\":false,\"queryType\":\"phrase\",\"value\":\"test-host\",\"valueType\":\"string\"}]]}}","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"742feb36-ac4c-45e0-b8a5-3b3cfa66b6d2","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"$state":{"store":"appState"},"meta":{"disabled":false,"key":"event.action","negate":false,"type":"phrase","params":{"query":"Process Create (rule: ProcessCreate)"}},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"alert_suppression":{"group_by":["process.parent.name"],"duration":{"value":5,"unit":"h"},"missing_fields_strategy":"suppress"},"actions":[]} +{"id":"e9430a4c-5fce-41b7-9d55-7645360e11d9","updated_at":"2025-08-18T03:40:30.081Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.326Z","created_by":"841510929","name":"test_kql_with_alert_suppression","tags":["child process","ms office"],"interval":"1h","enabled":true,"revision":1,"description":"Process started by MS Office program - possible payload","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-70m","rule_id":"2c6c5352-11cb-40a5-9294-e61ef5f1954f","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"query","language":"kuery","index":["logs-*"],"query":"process.parent.name:EXCEL.EXE or process.parent.name:MSPUB.EXE or process.parent.name:OUTLOOK.EXE or process.parent.name:POWERPNT.EXE or process.parent.name:VISIO.EXE or process.parent.name:WINWORD.EXE\n","filters":[{"meta":{"type":"phrase","key":"event.action","params":{"query":"Process Create (rule: ProcessCreate)"},"disabled":false,"negate":false},"$state":{"store":"appState"},"query":{"match_phrase":{"event.action":{"query":"Process Create (rule: ProcessCreate)"}}}}],"alert_suppression":{"group_by":["process.parent.name"],"duration":{"value":5,"unit":"h"},"missing_fields_strategy":"suppress"},"actions":[]} +{"id":"45241dcf-1bb2-41eb-8e91-89741af275c0","updated_at":"2025-08-18T03:43:41.240Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.317Z","created_by":"841510929","name":"test_eql_rule","tags":["EQL","Windows","rundll32.exe"],"interval":"5m","enabled":true,"revision":1,"description":"Unusual rundll32.exe network connection","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2cc8f325-e1b1-4201-8b8d-88a51c94992b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"event.type","type":"keyword","ecs":true},{"name":"process.args","type":"keyword","ecs":true},{"name":"process.args_count","type":"long","ecs":true},{"name":"process.entity_id","type":"keyword","ecs":true},{"name":"process.name","type":"keyword","ecs":true},{"name":"process.pe.original_file_name","type":"keyword","ecs":true}],"setup":"None","type":"eql","language":"eql","index":["logs-*"],"query":"sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n","filters":[],"actions":[]} +{"id":"11d7b970-0076-4ae1-b328-16d6778489f2","updated_at":"2025-08-18T03:45:34.509Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.308Z","created_by":"841510929","name":"test_esql_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Find Excel events","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"7e0f6dae-5847-465f-89e9-a6de0e9ef918","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"esql","language":"esql","query":"from auditbeat-* METADATA _id, _version, _index | KEEP process.parent.name | where process.parent.name == \"EXCEL.EXE\"\n","actions":[]} +{"id":"72abd101-fe39-43f0-a6d1-e9a373684cab","updated_at":"2025-08-18T03:46:00.515Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.334Z","created_by":"841510929","name":"test_new_terms_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Detects a user associated with a new IP address","risk_score":0,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2390c9dd-ad90-4af6-97a4-1d607ba0f092","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"user.id","type":"keyword","ecs":true},{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"new_terms","query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","new_terms_fields":["user.id","source.ip"],"history_window_start":"now-30d","index":["auditbeat*"],"filters":[],"language":"kuery","actions":[]} +{"id":"e0e31a34-2e18-40c0-af09-539021e8439d","updated_at":"2025-08-18T03:47:21.590Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.344Z","created_by":"841510929","name":"test_indicator_match_rule_with_email_actions","tags":[],"interval":"5m","enabled":true,"revision":5,"description":"Checks for bad IP addresses listed in the ip-threat-list index","risk_score":0,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator","actions":[{"id":"elastic-cloud-email","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","subject":"Test Actions","to":["tradebot-elastic@elastic.com"]},"action_type_id":".email","uuid":"74c388a4-c94f-4541-bacc-2a1b4c47e768","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} +{"id":"a0d623ea-e8a4-4eff-9c6c-643ceff9f3e5","updated_at":"2025-08-18T03:44:54.407Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.331Z","created_by":"841510929","name":"test_threshold_with_rule_exception","tags":["Brute force"],"interval":"2m","enabled":true,"revision":1,"description":"Detects when there are 20 or more failed login attempts from the same IP address with a 2 minute time frame.","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-3m","rule_id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[{"field":"source.geo.city_name","operator":"equals","severity":"low","value":"Manchester"},{"field":"source.geo.city_name","operator":"equals","severity":"medium","value":"London"},{"field":"source.geo.city_name","operator":"equals","severity":"high","value":"Birmingham"},{"field":"source.geo.city_name","operator":"equals","severity":"critical","value":"Wallingford"}],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"82395156-8ad2-46c3-be79-1f1a23c0d802","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","type":"rule_default","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"threshold","language":"kuery","index":["winlogbeat-*"],"query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","filters":[],"threshold":{"field":["source.ip"],"value":20,"cardinality":[]},"actions":[]} +{"id":"9bcffa42-d8b5-4706-afec-3cf33b19d9b1","updated_at":"2025-08-18T03:48:19.634Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.415Z","created_by":"841510929","name":"test_machine_learning_rule_with_index_action_connector ","tags":["machine learning","Linux"],"interval":"5m","enabled":true,"revision":5,"description":"Generates alerts when the job discovers anomalies over 70","risk_score":0,"severity":"high","note":"Shut down the internet.","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[],"setup":"This rule requires data coming in from Elastic Defend.","type":"machine_learning","anomaly_threshold":70,"machine_learning_job_id":["linux_anomalous_network_activity_ecs"],"actions":[{"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","params":{"documents":[{"rule.id":"{{rule.id}}"}]},"action_type_id":".index","uuid":"175f50f8-3bc1-4017-805f-e532d7eb2f91","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} {"_version":"WzE3NjU1LDhd","created_at":"2025-08-14T12:42:04.522Z","created_by":"841510929","description":"","id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","immutable":false,"list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","name":"Test Excpetion List","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"14b3565d-0c8a-48db-b76a-e46c01574a57","type":"detection","updated_at":"2025-08-14T12:42:04.522Z","updated_by":"841510929","version":1} {"_version":"WzE3NjU2LDhd","comments":[],"created_at":"2025-08-14T12:42:34.361Z","created_by":"841510929","description":"Exception list item","entries":[{"type":"match","field":"host.name","value":"test-host","operator":"included"}],"id":"dc084b23-4b9c-40c9-a172-77468ee2a4d9","item_id":"734852b6-b3bf-4942-8b3b-c058bd16088f","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","name":"host_excpetion","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"50c46edf-691b-4397-ad9e-e06a544a81d0","type":"simple","updated_at":"2025-08-14T12:42:34.361Z","updated_by":"841510929"} {"_version":"WzE3NjUwLDhd","created_at":"2025-08-14T12:19:29.454Z","created_by":"841510929","description":"Exception list containing exceptions for rule with id: 51a51212-5975-45ac-b909-c7840a903141","id":"82395156-8ad2-46c3-be79-1f1a23c0d802","immutable":false,"list_id":"0a4124f8-2074-450b-8689-d7dee319c666","name":"Exceptions for rule - Test Windows server prml-19","namespace_type":"single","os_types":[],"tags":["default_rule_exception_list"],"tie_breaker_id":"46a0d0b5-8793-4f60-a20b-6f76274b1722","type":"rule_default","updated_at":"2025-08-14T12:19:29.454Z","updated_by":"841510929","version":1} From 82606e8152b8ac21d7f29ee0ced99ccb143373b0 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 29 Oct 2025 11:31:55 -0400 Subject: [PATCH 12/21] Update for correct schema validation --- detection_rules/etc/custom-consolidated-rules.ndjson | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/detection_rules/etc/custom-consolidated-rules.ndjson b/detection_rules/etc/custom-consolidated-rules.ndjson index c6c969e0eb8..63db1a89c6f 100644 --- a/detection_rules/etc/custom-consolidated-rules.ndjson +++ b/detection_rules/etc/custom-consolidated-rules.ndjson @@ -4,9 +4,9 @@ {"id":"45241dcf-1bb2-41eb-8e91-89741af275c0","updated_at":"2025-08-18T03:43:41.240Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.317Z","created_by":"841510929","name":"test_eql_rule","tags":["EQL","Windows","rundll32.exe"],"interval":"5m","enabled":true,"revision":1,"description":"Unusual rundll32.exe network connection","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2cc8f325-e1b1-4201-8b8d-88a51c94992b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"event.type","type":"keyword","ecs":true},{"name":"process.args","type":"keyword","ecs":true},{"name":"process.args_count","type":"long","ecs":true},{"name":"process.entity_id","type":"keyword","ecs":true},{"name":"process.name","type":"keyword","ecs":true},{"name":"process.pe.original_file_name","type":"keyword","ecs":true}],"setup":"None","type":"eql","language":"eql","index":["logs-*"],"query":"sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n","filters":[],"actions":[]} {"id":"11d7b970-0076-4ae1-b328-16d6778489f2","updated_at":"2025-08-18T03:45:34.509Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.308Z","created_by":"841510929","name":"test_esql_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Find Excel events","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"7e0f6dae-5847-465f-89e9-a6de0e9ef918","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"esql","language":"esql","query":"from auditbeat-* METADATA _id, _version, _index | KEEP process.parent.name | where process.parent.name == \"EXCEL.EXE\"\n","actions":[]} {"id":"72abd101-fe39-43f0-a6d1-e9a373684cab","updated_at":"2025-08-18T03:46:00.515Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.334Z","created_by":"841510929","name":"test_new_terms_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Detects a user associated with a new IP address","risk_score":0,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2390c9dd-ad90-4af6-97a4-1d607ba0f092","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"user.id","type":"keyword","ecs":true},{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"new_terms","query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","new_terms_fields":["user.id","source.ip"],"history_window_start":"now-30d","index":["auditbeat*"],"filters":[],"language":"kuery","actions":[]} -{"id":"e0e31a34-2e18-40c0-af09-539021e8439d","updated_at":"2025-08-18T03:47:21.590Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.344Z","created_by":"841510929","name":"test_indicator_match_rule_with_email_actions","tags":[],"interval":"5m","enabled":true,"revision":5,"description":"Checks for bad IP addresses listed in the ip-threat-list index","risk_score":0,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator","actions":[{"id":"elastic-cloud-email","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","subject":"Test Actions","to":["tradebot-elastic@elastic.com"]},"action_type_id":".email","uuid":"74c388a4-c94f-4541-bacc-2a1b4c47e768","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} -{"id":"a0d623ea-e8a4-4eff-9c6c-643ceff9f3e5","updated_at":"2025-08-18T03:44:54.407Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.331Z","created_by":"841510929","name":"test_threshold_with_rule_exception","tags":["Brute force"],"interval":"2m","enabled":true,"revision":1,"description":"Detects when there are 20 or more failed login attempts from the same IP address with a 2 minute time frame.","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-3m","rule_id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[{"field":"source.geo.city_name","operator":"equals","severity":"low","value":"Manchester"},{"field":"source.geo.city_name","operator":"equals","severity":"medium","value":"London"},{"field":"source.geo.city_name","operator":"equals","severity":"high","value":"Birmingham"},{"field":"source.geo.city_name","operator":"equals","severity":"critical","value":"Wallingford"}],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"82395156-8ad2-46c3-be79-1f1a23c0d802","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","type":"rule_default","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"threshold","language":"kuery","index":["winlogbeat-*"],"query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","filters":[],"threshold":{"field":["source.ip"],"value":20,"cardinality":[]},"actions":[]} -{"id":"9bcffa42-d8b5-4706-afec-3cf33b19d9b1","updated_at":"2025-08-18T03:48:19.634Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.415Z","created_by":"841510929","name":"test_machine_learning_rule_with_index_action_connector ","tags":["machine learning","Linux"],"interval":"5m","enabled":true,"revision":5,"description":"Generates alerts when the job discovers anomalies over 70","risk_score":0,"severity":"high","note":"Shut down the internet.","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[],"setup":"This rule requires data coming in from Elastic Defend.","type":"machine_learning","anomaly_threshold":70,"machine_learning_job_id":["linux_anomalous_network_activity_ecs"],"actions":[{"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","params":{"documents":[{"rule.id":"{{rule.id}}"}]},"action_type_id":".index","uuid":"175f50f8-3bc1-4017-805f-e532d7eb2f91","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} +{"id":"2294573b-7afc-460d-b512-d119fe6f861e","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","name":"test_indicator_match_rule_with_email_actions","immutable":false,"rule_source":{"type":"internal"},"version":1,"revision":1,"updated_at":"2025-10-29T15:29:49.728Z","updated_by":"3610252053","created_at":"2025-10-29T15:29:07.740Z","created_by":"3610252053","enabled":true,"interval":"5m","from":"now-6m","to":"now","description":"Checks for bad IP addresses listed in the ip-threat-list index","tags":[],"author":["841510929"],"license":"","threat":[],"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","note":"None","false_positives":[],"references":[],"risk_score":0,"risk_score_mapping":[],"severity":"medium","severity_mapping":[],"output_index":"","max_signals":100,"exceptions_list":[],"actions":[{"id":"Elastic-Cloud-SMTP","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","to":["tradebot-elastic@elastic.com"],"subject":"Test Actions"},"action_type_id":".email","uuid":"98de9d3f-87e9-468a-b656-26f8c2f64c00","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}],"meta":{"kibana_siem_app_url":""},"type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator"} +{"id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","updated_at":"2025-08-18T03:44:54.407Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.331Z","created_by":"841510929","name":"test_threshold_with_rule_exception","tags":["Brute force"],"interval":"2m","enabled":true,"revision":1,"description":"Detects when there are 20 or more failed login attempts from the same IP address with a 2 minute time frame.","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-3m","rule_id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[{"field":"source.geo.city_name","operator":"equals","severity":"low","value":"Manchester"},{"field":"source.geo.city_name","operator":"equals","severity":"medium","value":"London"},{"field":"source.geo.city_name","operator":"equals","severity":"high","value":"Birmingham"},{"field":"source.geo.city_name","operator":"equals","severity":"critical","value":"Wallingford"}],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"82395156-8ad2-46c3-be79-1f1a23c0d802","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","type":"rule_default","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"threshold","language":"kuery","index":["winlogbeat-*"],"query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","filters":[],"threshold":{"field":["source.ip"],"value":20,"cardinality":[]},"actions":[]} +{"id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","updated_at":"2025-08-18T03:48:19.634Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.415Z","created_by":"841510929","name":"test_machine_learning_rule_with_index_action_connector ","tags":["machine learning","Linux"],"interval":"5m","enabled":true,"revision":5,"description":"Generates alerts when the job discovers anomalies over 70","risk_score":0,"severity":"high","note":"Shut down the internet.","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[],"setup":"This rule requires data coming in from Elastic Defend.","type":"machine_learning","anomaly_threshold":70,"machine_learning_job_id":["linux_anomalous_network_activity_ecs"],"actions":[{"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","params":{"documents":[{"rule.id":"{{rule.id}}"}]},"action_type_id":".index","uuid":"175f50f8-3bc1-4017-805f-e532d7eb2f91","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} {"_version":"WzE3NjU1LDhd","created_at":"2025-08-14T12:42:04.522Z","created_by":"841510929","description":"","id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","immutable":false,"list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","name":"Test Excpetion List","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"14b3565d-0c8a-48db-b76a-e46c01574a57","type":"detection","updated_at":"2025-08-14T12:42:04.522Z","updated_by":"841510929","version":1} {"_version":"WzE3NjU2LDhd","comments":[],"created_at":"2025-08-14T12:42:34.361Z","created_by":"841510929","description":"Exception list item","entries":[{"type":"match","field":"host.name","value":"test-host","operator":"included"}],"id":"dc084b23-4b9c-40c9-a172-77468ee2a4d9","item_id":"734852b6-b3bf-4942-8b3b-c058bd16088f","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","name":"host_excpetion","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"50c46edf-691b-4397-ad9e-e06a544a81d0","type":"simple","updated_at":"2025-08-14T12:42:34.361Z","updated_by":"841510929"} {"_version":"WzE3NjUwLDhd","created_at":"2025-08-14T12:19:29.454Z","created_by":"841510929","description":"Exception list containing exceptions for rule with id: 51a51212-5975-45ac-b909-c7840a903141","id":"82395156-8ad2-46c3-be79-1f1a23c0d802","immutable":false,"list_id":"0a4124f8-2074-450b-8689-d7dee319c666","name":"Exceptions for rule - Test Windows server prml-19","namespace_type":"single","os_types":[],"tags":["default_rule_exception_list"],"tie_breaker_id":"46a0d0b5-8793-4f60-a20b-6f76274b1722","type":"rule_default","updated_at":"2025-08-14T12:19:29.454Z","updated_by":"841510929","version":1} From 1cc060cbe002533db0144ae8f42638a501d2a397 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 29 Oct 2025 12:23:29 -0400 Subject: [PATCH 13/21] revert email change --- detection_rules/etc/custom-consolidated-rules.ndjson | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/etc/custom-consolidated-rules.ndjson b/detection_rules/etc/custom-consolidated-rules.ndjson index 63db1a89c6f..46cd98ca637 100644 --- a/detection_rules/etc/custom-consolidated-rules.ndjson +++ b/detection_rules/etc/custom-consolidated-rules.ndjson @@ -4,7 +4,7 @@ {"id":"45241dcf-1bb2-41eb-8e91-89741af275c0","updated_at":"2025-08-18T03:43:41.240Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.317Z","created_by":"841510929","name":"test_eql_rule","tags":["EQL","Windows","rundll32.exe"],"interval":"5m","enabled":true,"revision":1,"description":"Unusual rundll32.exe network connection","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2cc8f325-e1b1-4201-8b8d-88a51c94992b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"event.type","type":"keyword","ecs":true},{"name":"process.args","type":"keyword","ecs":true},{"name":"process.args_count","type":"long","ecs":true},{"name":"process.entity_id","type":"keyword","ecs":true},{"name":"process.name","type":"keyword","ecs":true},{"name":"process.pe.original_file_name","type":"keyword","ecs":true}],"setup":"None","type":"eql","language":"eql","index":["logs-*"],"query":"sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n","filters":[],"actions":[]} {"id":"11d7b970-0076-4ae1-b328-16d6778489f2","updated_at":"2025-08-18T03:45:34.509Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.308Z","created_by":"841510929","name":"test_esql_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Find Excel events","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"7e0f6dae-5847-465f-89e9-a6de0e9ef918","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"esql","language":"esql","query":"from auditbeat-* METADATA _id, _version, _index | KEEP process.parent.name | where process.parent.name == \"EXCEL.EXE\"\n","actions":[]} {"id":"72abd101-fe39-43f0-a6d1-e9a373684cab","updated_at":"2025-08-18T03:46:00.515Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.334Z","created_by":"841510929","name":"test_new_terms_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Detects a user associated with a new IP address","risk_score":0,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2390c9dd-ad90-4af6-97a4-1d607ba0f092","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"user.id","type":"keyword","ecs":true},{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"new_terms","query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","new_terms_fields":["user.id","source.ip"],"history_window_start":"now-30d","index":["auditbeat*"],"filters":[],"language":"kuery","actions":[]} -{"id":"2294573b-7afc-460d-b512-d119fe6f861e","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","name":"test_indicator_match_rule_with_email_actions","immutable":false,"rule_source":{"type":"internal"},"version":1,"revision":1,"updated_at":"2025-10-29T15:29:49.728Z","updated_by":"3610252053","created_at":"2025-10-29T15:29:07.740Z","created_by":"3610252053","enabled":true,"interval":"5m","from":"now-6m","to":"now","description":"Checks for bad IP addresses listed in the ip-threat-list index","tags":[],"author":["841510929"],"license":"","threat":[],"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","note":"None","false_positives":[],"references":[],"risk_score":0,"risk_score_mapping":[],"severity":"medium","severity_mapping":[],"output_index":"","max_signals":100,"exceptions_list":[],"actions":[{"id":"Elastic-Cloud-SMTP","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","to":["tradebot-elastic@elastic.com"],"subject":"Test Actions"},"action_type_id":".email","uuid":"98de9d3f-87e9-468a-b656-26f8c2f64c00","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}],"meta":{"kibana_siem_app_url":""},"type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator"} +{"id":"e0e31a34-2e18-40c0-af09-539021e8439d","updated_at":"2025-08-18T03:47:21.590Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.344Z","created_by":"841510929","name":"test_indicator_match_rule_with_email_actions","tags":[],"interval":"5m","enabled":true,"revision":5,"description":"Checks for bad IP addresses listed in the ip-threat-list index","risk_score":0,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator","actions":[{"id":"elastic-cloud-email","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","subject":"Test Actions","to":["tradebot-elastic@elastic.com"]},"action_type_id":".email","uuid":"74c388a4-c94f-4541-bacc-2a1b4c47e768","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} {"id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","updated_at":"2025-08-18T03:44:54.407Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.331Z","created_by":"841510929","name":"test_threshold_with_rule_exception","tags":["Brute force"],"interval":"2m","enabled":true,"revision":1,"description":"Detects when there are 20 or more failed login attempts from the same IP address with a 2 minute time frame.","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-3m","rule_id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[{"field":"source.geo.city_name","operator":"equals","severity":"low","value":"Manchester"},{"field":"source.geo.city_name","operator":"equals","severity":"medium","value":"London"},{"field":"source.geo.city_name","operator":"equals","severity":"high","value":"Birmingham"},{"field":"source.geo.city_name","operator":"equals","severity":"critical","value":"Wallingford"}],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"82395156-8ad2-46c3-be79-1f1a23c0d802","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","type":"rule_default","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"threshold","language":"kuery","index":["winlogbeat-*"],"query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","filters":[],"threshold":{"field":["source.ip"],"value":20,"cardinality":[]},"actions":[]} {"id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","updated_at":"2025-08-18T03:48:19.634Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.415Z","created_by":"841510929","name":"test_machine_learning_rule_with_index_action_connector ","tags":["machine learning","Linux"],"interval":"5m","enabled":true,"revision":5,"description":"Generates alerts when the job discovers anomalies over 70","risk_score":0,"severity":"high","note":"Shut down the internet.","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[],"setup":"This rule requires data coming in from Elastic Defend.","type":"machine_learning","anomaly_threshold":70,"machine_learning_job_id":["linux_anomalous_network_activity_ecs"],"actions":[{"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","params":{"documents":[{"rule.id":"{{rule.id}}"}]},"action_type_id":".index","uuid":"175f50f8-3bc1-4017-805f-e532d7eb2f91","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} {"_version":"WzE3NjU1LDhd","created_at":"2025-08-14T12:42:04.522Z","created_by":"841510929","description":"","id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","immutable":false,"list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","name":"Test Excpetion List","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"14b3565d-0c8a-48db-b76a-e46c01574a57","type":"detection","updated_at":"2025-08-14T12:42:04.522Z","updated_by":"841510929","version":1} From 85fcbc9fa7d23db2ba1dfbd71b2ff1414d9ce1fb Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Wed, 29 Oct 2025 13:14:39 -0400 Subject: [PATCH 14/21] Add email action connector --- detection_rules/etc/custom-consolidated-rules.ndjson | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/detection_rules/etc/custom-consolidated-rules.ndjson b/detection_rules/etc/custom-consolidated-rules.ndjson index 46cd98ca637..2efcb255f3f 100644 --- a/detection_rules/etc/custom-consolidated-rules.ndjson +++ b/detection_rules/etc/custom-consolidated-rules.ndjson @@ -4,7 +4,7 @@ {"id":"45241dcf-1bb2-41eb-8e91-89741af275c0","updated_at":"2025-08-18T03:43:41.240Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.317Z","created_by":"841510929","name":"test_eql_rule","tags":["EQL","Windows","rundll32.exe"],"interval":"5m","enabled":true,"revision":1,"description":"Unusual rundll32.exe network connection","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2cc8f325-e1b1-4201-8b8d-88a51c94992b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"event.type","type":"keyword","ecs":true},{"name":"process.args","type":"keyword","ecs":true},{"name":"process.args_count","type":"long","ecs":true},{"name":"process.entity_id","type":"keyword","ecs":true},{"name":"process.name","type":"keyword","ecs":true},{"name":"process.pe.original_file_name","type":"keyword","ecs":true}],"setup":"None","type":"eql","language":"eql","index":["logs-*"],"query":"sequence by process.entity_id with maxspan=2h [process where event.type in (\"start\", \"process_started\") and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\") and ((process.args == \"rundll32.exe\" and process.args_count == 1) or (process.args != \"rundll32.exe\" and process.args_count == 0))] [network where event.type == \"connection\" and (process.name == \"rundll32.exe\" or process.pe.original_file_name == \"rundll32.exe\")]\n","filters":[],"actions":[]} {"id":"11d7b970-0076-4ae1-b328-16d6778489f2","updated_at":"2025-08-18T03:45:34.509Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.308Z","created_by":"841510929","name":"test_esql_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Find Excel events","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"7e0f6dae-5847-465f-89e9-a6de0e9ef918","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"process.parent.name","type":"keyword","ecs":true}],"setup":"None","type":"esql","language":"esql","query":"from auditbeat-* METADATA _id, _version, _index | KEEP process.parent.name | where process.parent.name == \"EXCEL.EXE\"\n","actions":[]} {"id":"72abd101-fe39-43f0-a6d1-e9a373684cab","updated_at":"2025-08-18T03:46:00.515Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.334Z","created_by":"841510929","name":"test_new_terms_rule_with_shared_rule_exception","tags":[],"interval":"5m","enabled":true,"revision":2,"description":"Detects a user associated with a new IP address","risk_score":0,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"2390c9dd-ad90-4af6-97a4-1d607ba0f092","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","type":"detection","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"user.id","type":"keyword","ecs":true},{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"new_terms","query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","new_terms_fields":["user.id","source.ip"],"history_window_start":"now-30d","index":["auditbeat*"],"filters":[],"language":"kuery","actions":[]} -{"id":"e0e31a34-2e18-40c0-af09-539021e8439d","updated_at":"2025-08-18T03:47:21.590Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.344Z","created_by":"841510929","name":"test_indicator_match_rule_with_email_actions","tags":[],"interval":"5m","enabled":true,"revision":5,"description":"Checks for bad IP addresses listed in the ip-threat-list index","risk_score":0,"severity":"medium","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator","actions":[{"id":"elastic-cloud-email","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","subject":"Test Actions","to":["tradebot-elastic@elastic.com"]},"action_type_id":".email","uuid":"74c388a4-c94f-4541-bacc-2a1b4c47e768","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} +{"id":"a1605087-5c6f-4363-9686-ecd47e9c44b6","rule_id":"4c589d81-2622-4036-8cc7-372ea8f0e038","name":"test_indicator_match_rule_with_email_actions","immutable":false,"rule_source":{"type":"internal"},"version":1,"revision":1,"updated_at":"2025-10-29T17:02:23.823Z","updated_by":"3610252053","created_at":"2025-10-29T17:01:09.848Z","created_by":"3610252053","enabled":true,"interval":"5m","from":"now-6m","to":"now","description":"Checks for bad IP addresses listed in the ip-threat-list index","tags":[],"author":["841510929"],"license":"","threat":[],"related_integrations":[],"required_fields":[{"name":"destination.ip","type":"ip","ecs":true},{"name":"destination.port","type":"long","ecs":true},{"name":"host.ip","type":"ip","ecs":true}],"setup":"None","note":"None","false_positives":[],"references":[],"risk_score":0,"risk_score_mapping":[],"severity":"medium","severity_mapping":[],"output_index":"","max_signals":100,"exceptions_list":[],"actions":[{"id":"1b8d347f-2542-4390-85de-2653518311e2","params":{"message":"Rule {{context.rule.name}} generated {{state.signals_count}} alerts","to":["tradebot-elastic@elastic.com"],"subject":"Test Actions"},"action_type_id":".email","uuid":"98de9d3f-87e9-468a-b656-26f8c2f64c00","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}],"meta":{"kibana_siem_app_url":""},"type":"threat_match","language":"kuery","index":["packetbeat-*"],"query":"destination.ip:* or host.ip:*\n","filters":[],"threat_filters":[],"threat_query":"*:*","threat_mapping":[{"entries":[{"field":"destination.ip","type":"mapping","value":"destination.ip"},{"field":"destination.port","type":"mapping","value":"destination.port"}]},{"entries":[{"field":"source.ip","type":"mapping","value":"host.ip"}]}],"threat_language":"kuery","threat_index":["ip-threat-list"],"threat_indicator_path":"threat.indicator"} {"id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","updated_at":"2025-08-18T03:44:54.407Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.331Z","created_by":"841510929","name":"test_threshold_with_rule_exception","tags":["Brute force"],"interval":"2m","enabled":true,"revision":1,"description":"Detects when there are 20 or more failed login attempts from the same IP address with a 2 minute time frame.","risk_score":0,"severity":"low","note":"None","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-3m","rule_id":"d46a29ca-9b5b-4cbd-b11f-35c6b59f207b","max_signals":100,"risk_score_mapping":[],"severity_mapping":[{"field":"source.geo.city_name","operator":"equals","severity":"low","value":"Manchester"},{"field":"source.geo.city_name","operator":"equals","severity":"medium","value":"London"},{"field":"source.geo.city_name","operator":"equals","severity":"high","value":"Birmingham"},{"field":"source.geo.city_name","operator":"equals","severity":"critical","value":"Wallingford"}],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[{"id":"82395156-8ad2-46c3-be79-1f1a23c0d802","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","type":"rule_default","namespace_type":"single"}],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[{"name":"source.ip","type":"ip","ecs":true}],"setup":"None","type":"threshold","language":"kuery","index":["winlogbeat-*"],"query":"host.name:prml-19 and event.category:authentication and event.outcome:failure\n","filters":[],"threshold":{"field":["source.ip"],"value":20,"cardinality":[]},"actions":[]} {"id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","updated_at":"2025-08-18T03:48:19.634Z","updated_by":"841510929","created_at":"2025-08-14T13:09:02.415Z","created_by":"841510929","name":"test_machine_learning_rule_with_index_action_connector ","tags":["machine learning","Linux"],"interval":"5m","enabled":true,"revision":5,"description":"Generates alerts when the job discovers anomalies over 70","risk_score":0,"severity":"high","note":"Shut down the internet.","license":"","output_index":"","meta":{"kibana_siem_app_url":""},"author":["841510929"],"false_positives":[],"from":"now-6m","rule_id":"8a3296e2-4a74-4d51-b819-8d4e58377bf7","max_signals":100,"risk_score_mapping":[],"severity_mapping":[],"threat":[],"to":"now","references":[],"version":1,"exceptions_list":[],"immutable":false,"rule_source":{"type":"internal"},"related_integrations":[],"required_fields":[],"setup":"This rule requires data coming in from Elastic Defend.","type":"machine_learning","anomaly_threshold":70,"machine_learning_job_id":["linux_anomalous_network_activity_ecs"],"actions":[{"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","params":{"documents":[{"rule.id":"{{rule.id}}"}]},"action_type_id":".index","uuid":"175f50f8-3bc1-4017-805f-e532d7eb2f91","frequency":{"summary":true,"notifyWhen":"onActiveAlert","throttle":null},"group":"default"}]} {"_version":"WzE3NjU1LDhd","created_at":"2025-08-14T12:42:04.522Z","created_by":"841510929","description":"","id":"5c6a49d5-b3f1-42f7-b484-1a36462f3e06","immutable":false,"list_id":"1c8a1378-8f0d-4565-9ae0-abeeaf3981ca","name":"Test Excpetion List","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"14b3565d-0c8a-48db-b76a-e46c01574a57","type":"detection","updated_at":"2025-08-14T12:42:04.522Z","updated_by":"841510929","version":1} @@ -12,4 +12,5 @@ {"_version":"WzE3NjUwLDhd","created_at":"2025-08-14T12:19:29.454Z","created_by":"841510929","description":"Exception list containing exceptions for rule with id: 51a51212-5975-45ac-b909-c7840a903141","id":"82395156-8ad2-46c3-be79-1f1a23c0d802","immutable":false,"list_id":"0a4124f8-2074-450b-8689-d7dee319c666","name":"Exceptions for rule - Test Windows server prml-19","namespace_type":"single","os_types":[],"tags":["default_rule_exception_list"],"tie_breaker_id":"46a0d0b5-8793-4f60-a20b-6f76274b1722","type":"rule_default","updated_at":"2025-08-14T12:19:29.454Z","updated_by":"841510929","version":1} {"_version":"WzE3NjUxLDhd","comments":[],"created_at":"2025-08-14T12:19:31.919Z","created_by":"841510929","description":"Exception list item","entries":[{"type":"match","field":" host.name","value":"liv-win-ser","operator":"included"}],"id":"1a4a30ce-bbf2-483a-86a7-7af9ea4b562e","item_id":"9ed8fb85-d920-4759-ba47-8d273cbb55b6","list_id":"0a4124f8-2074-450b-8689-d7dee319c666","name":"int-ips","namespace_type":"single","os_types":[],"tags":[],"tie_breaker_id":"430065d9-8c30-40bf-a589-706ae5cc490d","type":"simple","updated_at":"2025-08-14T12:19:31.919Z","updated_by":"841510929"} {"id":"e1b418e7-78df-4042-bfb0-1cc5fb6f7a4e","type":"action","updated_at":"2025-08-14T12:30:20.229Z","created_at":"2025-08-14T12:30:20.229Z","version":"WzI3MDY1OSwxMF0=","attributes":{"actionTypeId":".index","name":"test-connector","isMissingSecrets":false,"config":{"index":"logs-connector","refresh":false,"executionTimeField":null},"secrets":{}},"references":[],"managed":false,"coreMigrationVersion":"8.8.0","typeMigrationVersion":"10.1.0"} -{"exported_count":14,"exported_rules_count":9,"missing_rules":[],"missing_rules_count":0,"exported_exception_list_count":2,"exported_exception_list_item_count":2,"missing_exception_list_item_count":0,"missing_exception_list_items":[],"missing_exception_lists":[],"missing_exception_lists_count":0,"exported_action_connector_count":1,"missing_action_connection_count":0,"missing_action_connections":[],"excluded_action_connection_count":0,"excluded_action_connections":[]} +{"id":"1b8d347f-2542-4390-85de-2653518311e2","type":"action","updated_at":"2025-10-29T17:02:17.533Z","created_at":"2025-10-29T17:02:17.533Z","version":"WzI4ODg2OTIsMV0=","attributes":{"actionTypeId":".email","name":"Elastic-Cloud-Email","isMissingSecrets":false,"config":{"from":"tradebot-elastic@elastic.com","service":"gmail","host":"smtp.gmail.com","port":465,"secure":true,"hasAuth":false,"tenantId":null,"clientId":null,"oauthTokenUrl":null},"secrets":{}},"references":[],"managed":false,"coreMigrationVersion":"8.8.0","typeMigrationVersion":"10.1.0"} +{"exported_count":15,"exported_rules_count":9,"missing_rules":[],"missing_rules_count":0,"exported_exception_list_count":2,"exported_exception_list_item_count":2,"missing_exception_list_item_count":0,"missing_exception_list_items":[],"missing_exception_lists":[],"missing_exception_lists_count":0,"exported_action_connector_count":2,"missing_action_connection_count":0,"missing_action_connections":[],"excluded_action_connection_count":0,"excluded_action_connections":[]} From 65830022d426e4eee9a360f70c8c3509be785cdd Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Thu, 6 Nov 2025 15:09:48 -0500 Subject: [PATCH 15/21] Update Non ECS Combined Mappings --- detection_rules/index_mappings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index 778a1b2e55c..b234c34b333 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -289,8 +289,11 @@ def get_filtered_index_schema( filtered_index_lookup.update(custom_mapping) # Reduce the combined mappings to only the matched indices (local schema validation source of truth) + # Custom and non-ecs mappings are filtered before being sent to this function in prepare mappings combined_mappings: dict[str, Any] = {} utils.combine_dicts(combined_mappings, deepcopy(ecs_schema)) + utils.combine_dicts(combined_mappings, deepcopy(non_ecs_mapping)) + utils.combine_dicts(combined_mappings, deepcopy(custom_mapping)) for match in matches: utils.combine_dicts(combined_mappings, deepcopy(filtered_index_lookup.get(match, {}))) From 11deb3a383f47839649f3a72fee7ada40c6da0bb Mon Sep 17 00:00:00 2001 From: Eric Forte <119343520+eric-forte-elastic@users.noreply.github.com> Date: Tue, 11 Nov 2025 09:48:45 -0500 Subject: [PATCH 16/21] Update detection_rules/rule_validators.py Co-authored-by: Mika Ayenson, PhD --- detection_rules/rule_validators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index e3a70d18bf8..792d7914dc4 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -662,7 +662,7 @@ def validate_query_text_with_schema( # noqa: PLR0913 # treat this target as non-fatal to honor EQL optional semantics. # To support EQL sequence and sub query validation we need to return this field to overwrite - # what would have been parsed via auto_add_field as the error message and query may be our of sync + # what would have been parsed via auto_add_field as the error message and query may be out of sync # depending on how the method is called. field = extract_error_field(query_text, exc) if ( From 356eda93d7ec3f2f30ec030a407f42e8b272be87 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 11 Nov 2025 09:58:59 -0500 Subject: [PATCH 17/21] fix typo --- detection_rules/rule_validators.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/detection_rules/rule_validators.py b/detection_rules/rule_validators.py index e3a70d18bf8..48dd178eb00 100644 --- a/detection_rules/rule_validators.py +++ b/detection_rules/rule_validators.py @@ -588,6 +588,8 @@ def add_stack_targets(query_text: str, include_endgame: bool) -> None: def validate(self, data: "QueryRuleData", meta: RuleMeta, max_attempts: int = 10) -> None: # type: ignore[reportIncompatibleMethodOverride] """Validate an EQL query using a unified plan of schema combinations.""" + # base field declaration + field = None if meta.query_schema_validation is False or meta.maturity == "deprecated": return @@ -662,7 +664,7 @@ def validate_query_text_with_schema( # noqa: PLR0913 # treat this target as non-fatal to honor EQL optional semantics. # To support EQL sequence and sub query validation we need to return this field to overwrite - # what would have been parsed via auto_add_field as the error message and query may be our of sync + # what would have been parsed via auto_add_field as the error message and query may be out of sync # depending on how the method is called. field = extract_error_field(query_text, exc) if ( From b8df2ac569c5cfab3cde9559cb9f9023c6b7f192 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 11 Nov 2025 11:33:18 -0500 Subject: [PATCH 18/21] Handle custom schema overwrites --- detection_rules/index_mappings.py | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index b234c34b333..10ca5e8b746 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -292,8 +292,6 @@ def get_filtered_index_schema( # Custom and non-ecs mappings are filtered before being sent to this function in prepare mappings combined_mappings: dict[str, Any] = {} utils.combine_dicts(combined_mappings, deepcopy(ecs_schema)) - utils.combine_dicts(combined_mappings, deepcopy(non_ecs_mapping)) - utils.combine_dicts(combined_mappings, deepcopy(custom_mapping)) for match in matches: utils.combine_dicts(combined_mappings, deepcopy(filtered_index_lookup.get(match, {}))) @@ -461,20 +459,29 @@ def prepare_mappings( # noqa: PLR0913 index_lookup.update(integration_index_lookup) # Load non-ecs schema and convert to index mapping format (nested schema) + # For non_ecs we need both a mapping and a schema as custom schemas can override non-ecs fields + # In these cases we need to accept the overwrite keep the original non-ecs field in the schema + non_ecs_schema: dict[str, Any] = {} non_ecs_mapping: dict[str, Any] = {} non_ecs = ecs.get_non_ecs_schema() for index in indices: - non_ecs_mapping.update(non_ecs.get(index, {})) - non_ecs_mapping = ecs.flatten(non_ecs_mapping) - non_ecs_mapping = utils.convert_to_nested_schema(non_ecs_mapping) + index_mapping = non_ecs.get(index, {}) + non_ecs_schema.update(index_mapping) + index_mapping = ecs.flatten(index_mapping) + index_mapping = utils.convert_to_nested_schema(index_mapping) + non_ecs_mapping.update({index: index_mapping}) + + non_ecs_schema = ecs.flatten(non_ecs_schema) + non_ecs_schema = utils.convert_to_nested_schema(non_ecs_schema) # Load custom schema and convert to index mapping format (nested schema) custom_mapping: dict[str, Any] = {} custom_indices = ecs.get_custom_schemas() for index in indices: - custom_mapping.update(custom_indices.get(index, {})) - custom_mapping = ecs.flatten(custom_mapping) - custom_mapping = utils.convert_to_nested_schema(custom_mapping) + index_mapping = custom_indices.get(index, {}) + index_mapping = ecs.flatten(index_mapping) + index_mapping = utils.convert_to_nested_schema(index_mapping) + custom_mapping.update({index: index_mapping}) # Load ECS in an index mapping format (nested schema) current_version = Version.parse(load_current_package_version(), optional_minor_and_patch=True) @@ -487,8 +494,8 @@ def prepare_mappings( # noqa: PLR0913 index_lookup.update({"rule-ecs-index": ecs_schema}) - if (not integration_mappings or existing_mappings) and not non_ecs_mapping and not ecs_schema: + if (not integration_mappings or existing_mappings) and not non_ecs_schema and not ecs_schema: raise ValueError("No mappings found") - index_lookup.update({"rule-non-ecs-index": non_ecs_mapping}) + index_lookup.update({"rule-non-ecs-index": non_ecs_schema}) return existing_mappings, index_lookup, combined_mappings From f6fc1c96b77f778f2e149efe648e533eb2c595d2 Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 11 Nov 2025 12:28:17 -0500 Subject: [PATCH 19/21] Add better comments --- detection_rules/index_mappings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index 10ca5e8b746..cdf2806e1aa 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -285,6 +285,8 @@ def get_filtered_index_schema( filtered_index_lookup = { key.replace("logs-endpoint.", "logs-endpoint.events."): value for key, value in filtered_index_lookup.items() } + # This overwrites any conflicts with non-ecs preferring what is defined in custom mappings + # This can be done safely as we have a specific non-ecs-index that will also be included with only non-ecs mappings filtered_index_lookup.update(non_ecs_mapping) filtered_index_lookup.update(custom_mapping) From df7a657d89a23ba74dbd6e2763682cf5e2208bca Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 11 Nov 2025 19:25:59 -0500 Subject: [PATCH 20/21] Support custom schema index overwrite --- detection_rules/index_mappings.py | 58 ++++++++++++++++++++----------- 1 file changed, 37 insertions(+), 21 deletions(-) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index cdf2806e1aa..bfd72da1b9f 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -159,6 +159,29 @@ def get_simulated_index_template_mappings(elastic_client: Elasticsearch, name: s return template["template"]["mappings"]["properties"] +def prune_mappings_of_unsupported_types( + integration: str, stream: str, stream_mappings: dict[str, Any], log: Callable[[str], None] +) -> dict[str, Any]: + """Prune fields with unsupported types (ES|QL) from the provided mappings.""" + nested_multifields = find_nested_multifields(stream_mappings) + for field in nested_multifields: + field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields" + log( + f"Warning: Nested multi-field `{field}` found in `{integration}-{stream}`. " + f"Removing parent field from schema for ES|QL validation." + ) + delete_nested_key_from_dict(stream_mappings, field_name) + nested_flattened_fields = find_flattened_fields_with_subfields(stream_mappings) + for field in nested_flattened_fields: + field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields" + log( + f"Warning: flattened field `{field}` found in `{integration}-{stream}` with sub fields. " + f"Removing parent field from schema for ES|QL validation." + ) + delete_nested_key_from_dict(stream_mappings, field_name) + return stream_mappings + + def prepare_integration_mappings( # noqa: PLR0913 rule_integrations: list[str], event_dataset_integrations: list[EventDataset], @@ -199,22 +222,7 @@ def prepare_integration_mappings( # noqa: PLR0913 for stream in package_schema: flat_schema = package_schema[stream] stream_mappings = flat_schema_to_index_mapping(flat_schema) - nested_multifields = find_nested_multifields(stream_mappings) - for field in nested_multifields: - field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields" - log( - f"Warning: Nested multi-field `{field}` found in `{integration}-{stream}`. " - f"Removing parent field from schema for ES|QL validation." - ) - delete_nested_key_from_dict(stream_mappings, field_name) - nested_flattened_fields = find_flattened_fields_with_subfields(stream_mappings) - for field in nested_flattened_fields: - field_name = str(field).split(".fields.")[0].replace(".", ".properties.") + ".fields" - log( - f"Warning: flattened field `{field}` found in `{integration}-{stream}` with sub fields. " - f"Removing parent field from schema for ES|QL validation." - ) - delete_nested_key_from_dict(stream_mappings, field_name) + stream_mappings = prune_mappings_of_unsupported_types(integration, stream, stream_mappings, log) utils.combine_dicts(integration_mappings, deepcopy(stream_mappings)) index_lookup[f"{integration}-{stream}"] = stream_mappings @@ -285,17 +293,19 @@ def get_filtered_index_schema( filtered_index_lookup = { key.replace("logs-endpoint.", "logs-endpoint.events."): value for key, value in filtered_index_lookup.items() } - # This overwrites any conflicts with non-ecs preferring what is defined in custom mappings - # This can be done safely as we have a specific non-ecs-index that will also be included with only non-ecs mappings - filtered_index_lookup.update(non_ecs_mapping) - filtered_index_lookup.update(custom_mapping) # Reduce the combined mappings to only the matched indices (local schema validation source of truth) # Custom and non-ecs mappings are filtered before being sent to this function in prepare mappings combined_mappings: dict[str, Any] = {} utils.combine_dicts(combined_mappings, deepcopy(ecs_schema)) for match in matches: - utils.combine_dicts(combined_mappings, deepcopy(filtered_index_lookup.get(match, {}))) + base = filtered_index_lookup.get(match, {}) + # Update filtered index with non-ecs and custom mappings + # Need to user a merge here to not overwrite existing fields + utils.combine_dicts(base, deepcopy(non_ecs_mapping.get(match, {}))) + utils.combine_dicts(base, deepcopy(custom_mapping.get(match, {}))) + filtered_index_lookup[match] = base + utils.combine_dicts(combined_mappings, deepcopy(base)) # Reduce the index lookup to only the matched indices (remote/Kibana schema validation source of truth) filtered_index_mapping: dict[str, Any] = {} @@ -473,8 +483,12 @@ def prepare_mappings( # noqa: PLR0913 index_mapping = utils.convert_to_nested_schema(index_mapping) non_ecs_mapping.update({index: index_mapping}) + # These need to be handled separately as we need to be able to validate non-ecs fields as a whole + # and also at a per index level as custom schemas can override non-ecs fields and/or indices non_ecs_schema = ecs.flatten(non_ecs_schema) non_ecs_schema = utils.convert_to_nested_schema(non_ecs_schema) + non_ecs_schema = prune_mappings_of_unsupported_types("non-ecs", "non-ecs", non_ecs_schema, log) + non_ecs_mapping = prune_mappings_of_unsupported_types("non-ecs", "non-ecs", non_ecs_mapping, log) # Load custom schema and convert to index mapping format (nested schema) custom_mapping: dict[str, Any] = {} @@ -484,6 +498,7 @@ def prepare_mappings( # noqa: PLR0913 index_mapping = ecs.flatten(index_mapping) index_mapping = utils.convert_to_nested_schema(index_mapping) custom_mapping.update({index: index_mapping}) + custom_mapping = prune_mappings_of_unsupported_types("custom", "custom", custom_mapping, log) # Load ECS in an index mapping format (nested schema) current_version = Version.parse(load_current_package_version(), optional_minor_and_patch=True) @@ -499,5 +514,6 @@ def prepare_mappings( # noqa: PLR0913 if (not integration_mappings or existing_mappings) and not non_ecs_schema and not ecs_schema: raise ValueError("No mappings found") index_lookup.update({"rule-non-ecs-index": non_ecs_schema}) + utils.combine_dicts(combined_mappings, deepcopy(non_ecs_schema)) return existing_mappings, index_lookup, combined_mappings From 24eeca4e2487f47c3314468ec948e35af067f10b Mon Sep 17 00:00:00 2001 From: eric-forte-elastic Date: Tue, 11 Nov 2025 19:32:01 -0500 Subject: [PATCH 21/21] fix typo --- detection_rules/index_mappings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/detection_rules/index_mappings.py b/detection_rules/index_mappings.py index bfd72da1b9f..bfcbd13316f 100644 --- a/detection_rules/index_mappings.py +++ b/detection_rules/index_mappings.py @@ -301,7 +301,7 @@ def get_filtered_index_schema( for match in matches: base = filtered_index_lookup.get(match, {}) # Update filtered index with non-ecs and custom mappings - # Need to user a merge here to not overwrite existing fields + # Need to use a merge here to not overwrite existing fields utils.combine_dicts(base, deepcopy(non_ecs_mapping.get(match, {}))) utils.combine_dicts(base, deepcopy(custom_mapping.get(match, {}))) filtered_index_lookup[match] = base