tomtan/memory-gateway
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Initial SOC memory POC implementation

Browse Source
This commit is contained in:
tomtan
2026-04-27 17:13:06 +08:00
parent fc68581198
commit e6b1520bce
89 changed files with 7610 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1001.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-1001",
"title": "Impossible travel login followed by MFA prompt fatigue",
"scenario": "o365_suspicious_login",
"alert_type": "azuread_impossible_travel",
"severity": "high",
"status": "confirmed",
"time_window": {"start": "2026-04-02T22:10:00+08:00", "end": "2026-04-02T23:30:00+08:00"},
"summary": "User account showed impossible travel between Shanghai and Amsterdam, followed by repeated MFA prompts and successful sign-in.",
"alert_source": "Microsoft Entra ID",
"entities": {"users": ["david@corp.example"], "hosts": ["WS-DAVID-01"], "mailboxes": ["david@corp.example"]},
"observables": {"ips": ["203.0.113.150", "198.51.100.61"], "domains": [], "urls": [], "hashes": []},
"evidence": ["Two successful sign-ins from geographically impossible locations within 15 minutes.", "MFA challenge volume increased abnormally before final success.", "User confirmed they did not initiate overseas login."],
"investigation_steps": ["Review sign-in logs and device IDs.", "Check MFA event sequence.", "Validate user travel status with manager."],
"conclusion": {"verdict": "true_positive", "reason": "Impossible travel plus user denial and MFA fatigue pattern.", "recommended_actions": ["Revoke sessions and reset credentials.", "Review mailbox rules and app consent."]},
"related_refs": {"playbooks": ["PB-O365-LOGIN-001"], "kb": ["KB-O365-IMPOSSIBLE-TRAVEL", "KB-O365-MFA-FATIGUE"], "cases": []},
"lessons_learned": ["Impossible travel needs to be combined with user confirmation and MFA telemetry."],
"tags": ["o365", "login", "impossible-travel", "mfa-fatigue"]
}

19
evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1002.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-1002",
"title": "Legacy protocol sign-in from unfamiliar IP blocked by policy",
"scenario": "o365_suspicious_login",
"alert_type": "azuread_legacy_auth_attempt",
"severity": "medium",
"status": "false_positive",
"time_window": {"start": "2026-04-04T07:50:00+08:00", "end": "2026-04-04T08:10:00+08:00"},
"summary": "Legacy authentication attempt from a cloud IP was blocked; investigation tied it to an approved migration tool test.",
"alert_source": "Microsoft Entra ID",
"entities": {"users": ["svc-migration@corp.example"], "hosts": [], "mailboxes": ["svc-migration@corp.example"]},
"observables": {"ips": ["192.0.2.24"], "domains": [], "urls": [], "hashes": []},
"evidence": ["The account is a known migration service account.", "Source IP matched approved cloud migration vendor range.", "No successful sign-in occurred due to policy block."],
"investigation_steps": ["Review service account inventory.", "Check change ticket for migration activity.", "Validate source IP against vendor allowlist."],
"conclusion": {"verdict": "false_positive", "reason": "Expected migration tool behavior with policy block and approved change window.", "recommended_actions": ["Tune alert suppression for approved migration windows."]},
"related_refs": {"playbooks": ["PB-O365-LOGIN-001"], "kb": ["KB-O365-LEGACY-AUTH"], "cases": []},
"lessons_learned": ["Service account context is essential before escalating legacy auth alerts."],
"tags": ["o365", "login", "false-positive", "legacy-auth"]
}

19
evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1003.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-1003",
"title": "Suspicious inbox rule creation after successful foreign login",
"scenario": "o365_suspicious_login",
"alert_type": "azuread_suspicious_inbox_rule_after_login",
"severity": "high",
"status": "confirmed",
"time_window": {"start": "2026-04-06T19:20:00+08:00", "end": "2026-04-06T20:45:00+08:00"},
"summary": "An overseas sign-in to Microsoft 365 was followed by inbox rule creation to hide finance-related emails.",
"alert_source": "Microsoft Defender for Cloud Apps",
"entities": {"users": ["emma@corp.example"], "hosts": ["WS-EMMA-07"], "mailboxes": ["emma@corp.example"]},
"observables": {"ips": ["198.51.100.98"], "domains": [], "urls": [], "hashes": []},
"evidence": ["Successful sign-in from untrusted ASN.", "Inbox rule moved wire transfer emails to RSS Feeds folder.", "Mailbox audit showed rule creation minutes after login."],
"investigation_steps": ["Review mailbox audit logs.", "Export suspicious inbox rules.", "Check for OAuth app consent and forwarding settings."],
"conclusion": {"verdict": "true_positive", "reason": "Account compromise indicators plus malicious inbox rule persistence.", "recommended_actions": ["Remove malicious rules.", "Reset account and revoke refresh tokens."]},
"related_refs": {"playbooks": ["PB-O365-LOGIN-001"], "kb": ["KB-O365-INBOX-RULE-ABUSE", "KB-O365-IMPOSSIBLE-TRAVEL"], "cases": []},
"lessons_learned": ["Mailbox rule inspection should be default for suspicious O365 login cases."],
"tags": ["o365", "login", "inbox-rule", "account-compromise"]
}

19
evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1004.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-1004",
"title": "Multiple failed logins from residential proxy but no successful access",
"scenario": "o365_suspicious_login",
"alert_type": "azuread_password_spray_attempt",
"severity": "medium",
"status": "pending",
"time_window": {"start": "2026-04-08T02:00:00+08:00", "end": "2026-04-08T03:10:00+08:00"},
"summary": "Repeated failed Microsoft 365 sign-in attempts targeted one user from a residential proxy network, with no successful authentication observed.",
"alert_source": "Microsoft Entra ID",
"entities": {"users": ["frank@corp.example"], "hosts": [], "mailboxes": ["frank@corp.example"]},
"observables": {"ips": ["203.0.113.201"], "domains": [], "urls": [], "hashes": []},
"evidence": ["High-volume failed attempts over a short period.", "Source IP attributed to a residential proxy provider.", "No matching successful sign-in or MFA event found."],
"investigation_steps": ["Check password spray pattern across tenant.", "Confirm user recent password reset history.", "Review conditional access outcomes."],
"conclusion": {"verdict": "uncertain", "reason": "Suspicious authentication pattern but no confirmed access or downstream activity.", "recommended_actions": ["Monitor account closely.", "Consider temporary sign-in risk remediation."]},
"related_refs": {"playbooks": ["PB-O365-LOGIN-001"], "kb": ["KB-O365-IMPOSSIBLE-TRAVEL"], "cases": []},
"lessons_learned": ["Pending cases should still capture reusable spray indicators without overcommitting verdict."],
"tags": ["o365", "login", "password-spray", "pending"]
}

19
evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1005.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-1005",
"title": "Traveling executive triggered impossible travel but activity was legitimate",
"scenario": "o365_suspicious_login",
"alert_type": "azuread_impossible_travel",
"severity": "medium",
"status": "false_positive",
"time_window": {"start": "2026-04-09T09:00:00+08:00", "end": "2026-04-09T09:40:00+08:00"},
"summary": "Executive account triggered impossible travel due to corporate VPN exit node while the user was on an approved overseas trip.",
"alert_source": "Microsoft Entra ID",
"entities": {"users": ["grace@corp.example"], "hosts": ["VIP-LAPTOP-01"], "mailboxes": ["grace@corp.example"]},
"observables": {"ips": ["192.0.2.90", "203.0.113.77"], "domains": [], "urls": [], "hashes": []},
"evidence": ["Approved travel request existed.", "One login originated from corporate VPN exit node.", "Device and user agent were consistent with known user profile."],
"investigation_steps": ["Check travel approval and itinerary.", "Review VPN egress mapping.", "Compare user agent and managed device posture."],
"conclusion": {"verdict": "false_positive", "reason": "Legitimate travel combined with VPN routing caused impossible travel signal.", "recommended_actions": ["Document travel context and improve analyst checklist."]},
"related_refs": {"playbooks": ["PB-O365-LOGIN-001"], "kb": ["KB-O365-IMPOSSIBLE-TRAVEL"], "cases": []},
"lessons_learned": ["Impossible travel should consider approved travel and VPN topology before escalation."],
"tags": ["o365", "login", "false-positive", "travel"]
}

19
evaluation/datasets/mock_cases/phishing/CASE-2026-0001.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-0001",
"title": "Finance user received invoice-themed phishing email",
"scenario": "phishing",
"alert_type": "mail_suspicious_attachment",
"severity": "high",
"status": "confirmed",
"time_window": {"start": "2026-04-01T09:10:00+08:00", "end": "2026-04-01T11:30:00+08:00"},
"summary": "Finance user received an invoice-themed phishing email containing a malicious HTML attachment that redirected to a credential harvesting page.",
"alert_source": "Secure Email Gateway",
"entities": {"users": ["alice@corp.example"], "hosts": ["FIN-LAPTOP-12"], "mailboxes": ["alice@corp.example"]},
"observables": {"sender_emails": ["billing@vendor-payments.com"], "domains": ["vendor-payments.com", "vendor-payments-login.com"], "urls": ["https://vendor-payments-login.com/review"], "ips": ["198.51.100.20"], "hashes": ["sha256:phish0001"]},
"evidence": ["Sender domain was newly observed and failed DMARC.", "Attachment redirected to a fake Microsoft 365 login page.", "User clicked the link before mail quarantine completed."],
"investigation_steps": ["Validate sender authentication results.", "Detonate HTML attachment in sandbox.", "Check mailbox click telemetry and account sign-in logs."],
"conclusion": {"verdict": "true_positive", "reason": "Aligned phishing indicators and confirmed click behavior.", "recommended_actions": ["Reset impacted account password.", "Block sender domain and landing URL.", "Hunt for similar emails in tenant."]},
"related_refs": {"playbooks": ["PB-PHISH-001"], "kb": ["KB-PHISH-HEADER-CHECK", "KB-CRED-HARVEST-PATTERNS"], "cases": []},
"lessons_learned": ["Invoice lure remains effective against finance users."],
"tags": ["phishing", "email", "credential-harvest", "finance"]
}

19
evaluation/datasets/mock_cases/phishing/CASE-2026-0002.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-0002",
"title": "Payroll notification email flagged but determined benign",
"scenario": "phishing",
"alert_type": "mail_suspicious_link",
"severity": "medium",
"status": "false_positive",
"time_window": {"start": "2026-04-03T08:40:00+08:00", "end": "2026-04-03T09:20:00+08:00"},
"summary": "Payroll update email was flagged due to a shortened URL, but the destination was the approved HR vendor portal.",
"alert_source": "Secure Email Gateway",
"entities": {"users": ["bob@corp.example"], "hosts": ["HR-LAPTOP-03"], "mailboxes": ["bob@corp.example"]},
"observables": {"sender_emails": ["notify@hr-vendor.example"], "domains": ["hr-vendor.example"], "urls": ["https://bit.ly/hr-portal-example"], "ips": [], "hashes": []},
"evidence": ["Sender domain aligned with SPF and DKIM.", "Destination domain matched approved supplier inventory.", "No credential prompt anomaly observed."],
"investigation_steps": ["Expand shortened URL.", "Validate vendor domain against allowlist.", "Review prior communication pattern with HR users."],
"conclusion": {"verdict": "false_positive", "reason": "Trusted vendor communication with expected destination.", "recommended_actions": ["Tune mail rule to reduce noisy alerts for approved HR vendor."]},
"related_refs": {"playbooks": ["PB-PHISH-001"], "kb": ["KB-PHISH-HEADER-CHECK"], "cases": []},
"lessons_learned": ["Short URLs alone should not drive phishing conclusion without destination validation."],
"tags": ["phishing", "email", "false-positive", "vendor"]
}

19
evaluation/datasets/mock_cases/phishing/CASE-2026-0003.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-0003",
"title": "Executive impersonation email requested urgent wire transfer",
"scenario": "phishing",
"alert_type": "mail_bec_impersonation",
"severity": "high",
"status": "confirmed",
"time_window": {"start": "2026-04-05T13:15:00+08:00", "end": "2026-04-05T15:00:00+08:00"},
"summary": "An executive impersonation email targeted finance staff with an urgent wire transfer request from a lookalike domain.",
"alert_source": "Secure Email Gateway",
"entities": {"users": ["carol@corp.example"], "hosts": ["FIN-LAPTOP-08"], "mailboxes": ["carol@corp.example"]},
"observables": {"sender_emails": ["ceo@c0rp-example.com"], "domains": ["c0rp-example.com"], "urls": [], "ips": ["203.0.113.45"], "hashes": []},
"evidence": ["Lookalike domain used numeric substitution.", "Language pressure matched prior BEC pattern.", "No historical communication from sender domain."],
"investigation_steps": ["Compare sender domain with corporate domain.", "Review historical communication graph.", "Confirm with executive assistant out of band."],
"conclusion": {"verdict": "true_positive", "reason": "Strong BEC indicators and confirmed spoofed sender identity.", "recommended_actions": ["Block sender domain.", "Notify finance team and update awareness content."]},
"related_refs": {"playbooks": ["PB-PHISH-001"], "kb": ["KB-CRED-HARVEST-PATTERNS"], "cases": []},
"lessons_learned": ["Lookalike domains need strong entity normalization in retrieval and detection logic."],
"tags": ["phishing", "bec", "executive-impersonation"]
}

19
evaluation/datasets/mock_cases/phishing/CASE-2026-0004.json Normal file
View File

@ -0,0 +1,19 @@
{
"case_id": "CASE-2026-0004",
"title": "Shared mailbox received OneDrive lure with HTML attachment",
"scenario": "phishing",
"alert_type": "mail_suspicious_attachment",
"severity": "medium",
"status": "confirmed",
"time_window": {"start": "2026-04-07T10:00:00+08:00", "end": "2026-04-07T12:05:00+08:00"},
"summary": "Shared finance mailbox received a fake OneDrive notification with an HTML attachment that led to credential collection.",
"alert_source": "Secure Email Gateway",
"entities": {"users": ["shared-finance@corp.example"], "hosts": [], "mailboxes": ["shared-finance@corp.example"]},
"observables": {"sender_emails": ["noreply@sharepoint-notify.com"], "domains": ["sharepoint-notify.com"], "urls": ["https://onedrive-review-login.example"], "ips": ["198.51.100.87"], "hashes": ["sha256:phish0004"]},
"evidence": ["Attachment rendered a fake Microsoft sign-in page.", "Landing page hosted outside Microsoft IP space.", "Mail body reused branding from previous phishing campaign."],
"investigation_steps": ["Render attachment safely.", "Review URL hosting provider reputation.", "Search tenant for same subject and sender."],
"conclusion": {"verdict": "true_positive", "reason": "Credential harvesting lure with campaign reuse indicators.", "recommended_actions": ["Block sender and URL.", "Search and purge duplicate emails."]},
"related_refs": {"playbooks": ["PB-PHISH-001"], "kb": ["KB-CRED-HARVEST-PATTERNS"], "cases": ["CASE-2026-0001"]},
"lessons_learned": ["Campaign reuse makes historical phishing similarity especially valuable."],
"tags": ["phishing", "email", "onedrive-lure"]
}

15
evaluation/datasets/mock_kb/kb/KB-CRED-HARVEST-PATTERNS.json Normal file
View File

@ -0,0 +1,15 @@
{
"doc_id": "KB-CRED-HARVEST-PATTERNS",
"doc_type": "kb",
"title": "Credential Harvesting Indicators",
"scenario": "phishing",
"summary": "Common indicators that a phishing case involves credential harvesting rather than simple spam or benign mail.",
"applicability": ["mail_suspicious_attachment", "mail_suspicious_link"],
"key_points": ["Landing page mimics Microsoft 365 or common SaaS login pages.", "HTML attachment often acts as a redirector rather than containing malware.", "Credential harvest campaigns frequently reuse branding and lures across tenants."],
"investigation_guidance": ["Capture full redirect chain.", "Look for post-click login anomalies in identity logs.", "Search for same lure across multiple mailboxes."],
"decision_points": ["User click plus sign-in anomaly greatly increases confidence.", "Branding reuse can help link separate phishing cases into one campaign."],
"related_entities": {"ttps": ["T1566.002"], "iocs": []},
"related_refs": {"playbooks": ["PB-PHISH-001"], "cases": []},
"tags": ["kb", "phishing", "credential-harvest"],
"updated_at": "2026-04-10T09:25:00+08:00"
}

15
evaluation/datasets/mock_kb/kb/KB-O365-IMPOSSIBLE-TRAVEL.json Normal file
View File

@ -0,0 +1,15 @@
{
"doc_id": "KB-O365-IMPOSSIBLE-TRAVEL",
"doc_type": "kb",
"title": "Interpreting O365 Impossible Travel Alerts",
"scenario": "o365_suspicious_login",
"summary": "Guidance for validating impossible travel alerts, including VPN, proxy, and approved travel false-positive conditions.",
"applicability": ["azuread_impossible_travel"],
"key_points": ["Impossible travel must be validated against user travel context.", "VPN egress and cloud proxy routing are common false-positive sources.", "Pair sign-in anomaly with MFA, mailbox, or device anomalies before concluding compromise."],
"investigation_guidance": ["Validate source ASN and IP history.", "Check user-approved travel or remote work context.", "Compare device ID and user agent consistency."],
"decision_points": ["User denial of travel plus new device strongly increases confidence.", "Approved travel and trusted VPN topology reduce confidence."],
"related_entities": {"ttps": ["T1078"], "iocs": []},
"related_refs": {"playbooks": ["PB-O365-LOGIN-001"], "cases": []},
"tags": ["kb", "o365", "impossible-travel"],
"updated_at": "2026-04-10T09:30:00+08:00"
}

15
evaluation/datasets/mock_kb/kb/KB-O365-INBOX-RULE-ABUSE.json Normal file
View File

@ -0,0 +1,15 @@
{
"doc_id": "KB-O365-INBOX-RULE-ABUSE",
"doc_type": "kb",
"title": "Inbox Rule Abuse After Account Compromise",
"scenario": "o365_suspicious_login",
"summary": "Common mailbox persistence behaviors after O365 account compromise, especially rule creation to hide or forward finance emails.",
"applicability": ["azuread_suspicious_inbox_rule_after_login"],
"key_points": ["Attackers often hide financial emails using move-to-folder rules.", "Forwarding and delete rules are strong post-compromise indicators.", "Mailbox audit logs should be reviewed immediately after suspicious login confirmation."],
"investigation_guidance": ["Enumerate all inbox rules and forwarding settings.", "Check mailbox audit timeline around suspicious sign-in.", "Review OAuth consents if inbox rules are absent but suspicious mail actions continue."],
"decision_points": ["Inbox rule creation shortly after suspicious login strongly supports compromise verdict."],
"related_entities": {"ttps": ["T1114"], "iocs": []},
"related_refs": {"playbooks": ["PB-O365-LOGIN-001"], "cases": []},
"tags": ["kb", "o365", "inbox-rule"],
"updated_at": "2026-04-10T09:40:00+08:00"
}

15
evaluation/datasets/mock_kb/kb/KB-O365-MFA-FATIGUE.json Normal file
View File

@ -0,0 +1,15 @@
{
"doc_id": "KB-O365-MFA-FATIGUE",
"doc_type": "kb",
"title": "MFA Fatigue Detection Notes",
"scenario": "o365_suspicious_login",
"summary": "Patterns for identifying MFA fatigue / push bombing during account compromise attempts.",
"applicability": ["azuread_impossible_travel", "azuread_suspicious_login"],
"key_points": ["Repeated MFA prompts preceding one successful prompt is suspicious.", "User-reported prompt fatigue is strong supporting evidence.", "MFA fatigue is often coupled with credential theft rather than password spray alone."],
"investigation_guidance": ["Review MFA event counts and timing.", "Check if the user acknowledged unexpected prompts.", "Look for subsequent session hijacking or mailbox abuse."],
"decision_points": ["Prompt flood plus user denial usually warrants immediate containment."],
"related_entities": {"ttps": ["T1621"], "iocs": []},
"related_refs": {"playbooks": ["PB-O365-LOGIN-001"], "cases": []},
"tags": ["kb", "o365", "mfa-fatigue"],
"updated_at": "2026-04-10T09:35:00+08:00"
}

15
evaluation/datasets/mock_kb/kb/KB-PHISH-HEADER-CHECK.json Normal file
View File

@ -0,0 +1,15 @@
{
"doc_id": "KB-PHISH-HEADER-CHECK",
"doc_type": "kb",
"title": "Phishing Header Validation Checklist",
"scenario": "phishing",
"summary": "Checklist for validating sender identity, domain reputation, and authentication results in suspected phishing emails.",
"applicability": ["mail_suspicious_attachment", "mail_suspicious_link", "mail_bec_impersonation"],
"key_points": ["Review SPF, DKIM, and DMARC alignment.", "Compare display name, envelope sender, and reply-to anomalies.", "Check domain age and known-good communication history."],
"investigation_guidance": ["Use message trace and header parser.", "Compare sender domain with vendor allowlist.", "Escalate lookalike domains even when content appears business-relevant."],
"decision_points": ["Newly observed domains with failed auth are high-risk.", "Benign vendor mail often has consistent historical sending patterns."],
"related_entities": {"ttps": ["T1566.001"], "iocs": []},
"related_refs": {"playbooks": ["PB-PHISH-001"], "cases": []},
"tags": ["kb", "phishing", "email-header"],
"updated_at": "2026-04-10T09:20:00+08:00"
}

15
evaluation/datasets/mock_kb/playbooks/PB-O365-LOGIN-001.json Normal file
View File

@ -0,0 +1,15 @@
{
"doc_id": "PB-O365-LOGIN-001",
"doc_type": "playbook",
"title": "O365 Suspicious Login Investigation Playbook",
"scenario": "o365_suspicious_login",
"summary": "Standard investigation steps for suspicious Entra ID sign-ins, impossible travel, MFA abuse, and follow-on mailbox abuse.",
"applicability": ["azuread_impossible_travel", "azuread_legacy_auth_attempt", "azuread_suspicious_inbox_rule_after_login", "azuread_password_spray_attempt"],
"key_points": ["Confirm user travel and business context.", "Review sign-in logs, device IDs, and user agents.", "Inspect downstream actions such as inbox rules, app consent, and forwarding."],
"investigation_guidance": ["Correlate MFA telemetry with sign-in sequence.", "Check risky sign-ins and risky users views.", "Revoke sessions and reset credentials when compromise is confirmed."],
"decision_points": ["Impossible travel alone is insufficient without corroborating evidence.", "Inbox rule creation after foreign login strongly increases confidence of compromise."],
"related_entities": {"ttps": ["T1078"], "iocs": []},
"related_refs": {"kb": ["KB-O365-IMPOSSIBLE-TRAVEL", "KB-O365-MFA-FATIGUE", "KB-O365-INBOX-RULE-ABUSE"], "cases": []},
"tags": ["playbook", "o365", "login"],
"updated_at": "2026-04-10T09:10:00+08:00"
}

15
evaluation/datasets/mock_kb/playbooks/PB-PHISH-001.json Normal file
View File

@ -0,0 +1,15 @@
{
"doc_id": "PB-PHISH-001",
"doc_type": "playbook",
"title": "Phishing Email Investigation Playbook",
"scenario": "phishing",
"summary": "Standard investigation steps for suspicious email, credential harvesting, and BEC-like cases.",
"applicability": ["mail_suspicious_attachment", "mail_suspicious_link", "mail_bec_impersonation"],
"key_points": ["Validate sender authentication results.", "Inspect landing URL and attachment behavior.", "Check whether the user clicked or submitted credentials."],
"investigation_guidance": ["Query email telemetry for same sender, subject, or URL.", "Review mailbox click logs and endpoint browser artifacts.", "Reset credentials if submission is suspected."],
"decision_points": ["If sender auth fails and user interaction exists, treat as likely phishing.", "If destination is allowlisted and communication pattern is expected, investigate false positive path."],
"related_entities": {"ttps": ["T1566"], "iocs": []},
"related_refs": {"kb": ["KB-PHISH-HEADER-CHECK", "KB-CRED-HARVEST-PATTERNS"], "cases": []},
"tags": ["playbook", "phishing", "email"],
"updated_at": "2026-04-10T09:00:00+08:00"
}

65
evaluation/datasets/normalized_cases/CASE-2026-0001.json Normal file
View File

@ -0,0 +1,65 @@
{
"id": "CASE-2026-0001",
"memory_type": "case",
"scenario": "phishing",
"title": "Finance user received invoice-themed phishing email",
"abstract": "Finance user received an invoice-themed phishing email containing a malicious HTML attachment that redirected to a credential harvesting page.",
"verdict": "true_positive",
"severity": "high",
"entities": {
"users": [
"alice@corp.example"
],
"hosts": [
"FIN-LAPTOP-12"
],
"mailboxes": [
"alice@corp.example"
]
},
"observables": {
"sender_emails": [
"billing@vendor-payments.com"
],
"domains": [
"vendor-payments.com",
"vendor-payments-login.com"
],
"urls": [
"https://vendor-payments-login.com/review"
],
"ips": [
"198.51.100.20"
],
"hashes": [
"sha256:phish0001"
]
},
"evidence": [
"Sender domain was newly observed and failed DMARC.",
"Attachment redirected to a fake Microsoft 365 login page.",
"User clicked the link before mail quarantine completed."
],
"patterns": [
"verdict:true_positive",
"scenario:phishing",
"alert_type:mail_suspicious_attachment"
],
"related_refs": {
"playbooks": [
"PB-PHISH-001"
],
"kb": [
"KB-PHISH-HEADER-CHECK",
"KB-CRED-HARVEST-PATTERNS"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/phishing/CASE-2026-0001.json",
"tags": [
"phishing",
"email",
"credential-harvest",
"finance"
]
}

59
evaluation/datasets/normalized_cases/CASE-2026-0002.json Normal file
View File

@ -0,0 +1,59 @@
{
"id": "CASE-2026-0002",
"memory_type": "case",
"scenario": "phishing",
"title": "Payroll notification email flagged but determined benign",
"abstract": "Payroll update email was flagged due to a shortened URL, but the destination was the approved HR vendor portal.",
"verdict": "false_positive",
"severity": "medium",
"entities": {
"users": [
"bob@corp.example"
],
"hosts": [
"HR-LAPTOP-03"
],
"mailboxes": [
"bob@corp.example"
]
},
"observables": {
"sender_emails": [
"notify@hr-vendor.example"
],
"domains": [
"hr-vendor.example"
],
"urls": [
"https://bit.ly/hr-portal-example"
],
"ips": [],
"hashes": []
},
"evidence": [
"Sender domain aligned with SPF and DKIM.",
"Destination domain matched approved supplier inventory.",
"No credential prompt anomaly observed."
],
"patterns": [
"verdict:false_positive",
"scenario:phishing",
"alert_type:mail_suspicious_link"
],
"related_refs": {
"playbooks": [
"PB-PHISH-001"
],
"kb": [
"KB-PHISH-HEADER-CHECK"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/phishing/CASE-2026-0002.json",
"tags": [
"phishing",
"email",
"false-positive",
"vendor"
]
}

58
evaluation/datasets/normalized_cases/CASE-2026-0003.json Normal file
View File

@ -0,0 +1,58 @@
{
"id": "CASE-2026-0003",
"memory_type": "case",
"scenario": "phishing",
"title": "Executive impersonation email requested urgent wire transfer",
"abstract": "An executive impersonation email targeted finance staff with an urgent wire transfer request from a lookalike domain.",
"verdict": "true_positive",
"severity": "high",
"entities": {
"users": [
"carol@corp.example"
],
"hosts": [
"FIN-LAPTOP-08"
],
"mailboxes": [
"carol@corp.example"
]
},
"observables": {
"sender_emails": [
"ceo@c0rp-example.com"
],
"domains": [
"c0rp-example.com"
],
"urls": [],
"ips": [
"203.0.113.45"
],
"hashes": []
},
"evidence": [
"Lookalike domain used numeric substitution.",
"Language pressure matched prior BEC pattern.",
"No historical communication from sender domain."
],
"patterns": [
"verdict:true_positive",
"scenario:phishing",
"alert_type:mail_bec_impersonation"
],
"related_refs": {
"playbooks": [
"PB-PHISH-001"
],
"kb": [
"KB-CRED-HARVEST-PATTERNS"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/phishing/CASE-2026-0003.json",
"tags": [
"phishing",
"bec",
"executive-impersonation"
]
}

62
evaluation/datasets/normalized_cases/CASE-2026-0004.json Normal file
View File

@ -0,0 +1,62 @@
{
"id": "CASE-2026-0004",
"memory_type": "case",
"scenario": "phishing",
"title": "Shared mailbox received OneDrive lure with HTML attachment",
"abstract": "Shared finance mailbox received a fake OneDrive notification with an HTML attachment that led to credential collection.",
"verdict": "true_positive",
"severity": "medium",
"entities": {
"users": [
"shared-finance@corp.example"
],
"hosts": [],
"mailboxes": [
"shared-finance@corp.example"
]
},
"observables": {
"sender_emails": [
"noreply@sharepoint-notify.com"
],
"domains": [
"sharepoint-notify.com"
],
"urls": [
"https://onedrive-review-login.example"
],
"ips": [
"198.51.100.87"
],
"hashes": [
"sha256:phish0004"
]
},
"evidence": [
"Attachment rendered a fake Microsoft sign-in page.",
"Landing page hosted outside Microsoft IP space.",
"Mail body reused branding from previous phishing campaign."
],
"patterns": [
"verdict:true_positive",
"scenario:phishing",
"alert_type:mail_suspicious_attachment"
],
"related_refs": {
"playbooks": [
"PB-PHISH-001"
],
"kb": [
"KB-CRED-HARVEST-PATTERNS"
],
"cases": [
"CASE-2026-0001"
]
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/phishing/CASE-2026-0004.json",
"tags": [
"phishing",
"email",
"onedrive-lure"
]
}

56
evaluation/datasets/normalized_cases/CASE-2026-1001.json Normal file
View File

@ -0,0 +1,56 @@
{
"id": "CASE-2026-1001",
"memory_type": "case",
"scenario": "o365_suspicious_login",
"title": "Impossible travel login followed by MFA prompt fatigue",
"abstract": "User account showed impossible travel between Shanghai and Amsterdam, followed by repeated MFA prompts and successful sign-in.",
"verdict": "true_positive",
"severity": "high",
"entities": {
"users": [
"david@corp.example"
],
"hosts": [
"WS-DAVID-01"
],
"mailboxes": [
"david@corp.example"
]
},
"observables": {
"ips": [
"203.0.113.150",
"198.51.100.61"
],
"domains": [],
"urls": [],
"hashes": []
},
"evidence": [
"Two successful sign-ins from geographically impossible locations within 15 minutes.",
"MFA challenge volume increased abnormally before final success.",
"User confirmed they did not initiate overseas login."
],
"patterns": [
"verdict:true_positive",
"scenario:o365_suspicious_login",
"alert_type:azuread_impossible_travel"
],
"related_refs": {
"playbooks": [
"PB-O365-LOGIN-001"
],
"kb": [
"KB-O365-IMPOSSIBLE-TRAVEL",
"KB-O365-MFA-FATIGUE"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1001.json",
"tags": [
"o365",
"login",
"impossible-travel",
"mfa-fatigue"
]
}

52
evaluation/datasets/normalized_cases/CASE-2026-1002.json Normal file
View File

@ -0,0 +1,52 @@
{
"id": "CASE-2026-1002",
"memory_type": "case",
"scenario": "o365_suspicious_login",
"title": "Legacy protocol sign-in from unfamiliar IP blocked by policy",
"abstract": "Legacy authentication attempt from a cloud IP was blocked; investigation tied it to an approved migration tool test.",
"verdict": "false_positive",
"severity": "medium",
"entities": {
"users": [
"svc-migration@corp.example"
],
"hosts": [],
"mailboxes": [
"svc-migration@corp.example"
]
},
"observables": {
"ips": [
"192.0.2.24"
],
"domains": [],
"urls": [],
"hashes": []
},
"evidence": [
"The account is a known migration service account.",
"Source IP matched approved cloud migration vendor range.",
"No successful sign-in occurred due to policy block."
],
"patterns": [
"verdict:false_positive",
"scenario:o365_suspicious_login",
"alert_type:azuread_legacy_auth_attempt"
],
"related_refs": {
"playbooks": [
"PB-O365-LOGIN-001"
],
"kb": [
"KB-O365-LEGACY-AUTH"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1002.json",
"tags": [
"o365",
"login",
"false-positive",
"legacy-auth"
]
}

55
evaluation/datasets/normalized_cases/CASE-2026-1003.json Normal file
View File

@ -0,0 +1,55 @@
{
"id": "CASE-2026-1003",
"memory_type": "case",
"scenario": "o365_suspicious_login",
"title": "Suspicious inbox rule creation after successful foreign login",
"abstract": "An overseas sign-in to Microsoft 365 was followed by inbox rule creation to hide finance-related emails.",
"verdict": "true_positive",
"severity": "high",
"entities": {
"users": [
"emma@corp.example"
],
"hosts": [
"WS-EMMA-07"
],
"mailboxes": [
"emma@corp.example"
]
},
"observables": {
"ips": [
"198.51.100.98"
],
"domains": [],
"urls": [],
"hashes": []
},
"evidence": [
"Successful sign-in from untrusted ASN.",
"Inbox rule moved wire transfer emails to RSS Feeds folder.",
"Mailbox audit showed rule creation minutes after login."
],
"patterns": [
"verdict:true_positive",
"scenario:o365_suspicious_login",
"alert_type:azuread_suspicious_inbox_rule_after_login"
],
"related_refs": {
"playbooks": [
"PB-O365-LOGIN-001"
],
"kb": [
"KB-O365-INBOX-RULE-ABUSE",
"KB-O365-IMPOSSIBLE-TRAVEL"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1003.json",
"tags": [
"o365",
"login",
"inbox-rule",
"account-compromise"
]
}

52
evaluation/datasets/normalized_cases/CASE-2026-1004.json Normal file
View File

@ -0,0 +1,52 @@
{
"id": "CASE-2026-1004",
"memory_type": "case",
"scenario": "o365_suspicious_login",
"title": "Multiple failed logins from residential proxy but no successful access",
"abstract": "Repeated failed Microsoft 365 sign-in attempts targeted one user from a residential proxy network, with no successful authentication observed.",
"verdict": "uncertain",
"severity": "medium",
"entities": {
"users": [
"frank@corp.example"
],
"hosts": [],
"mailboxes": [
"frank@corp.example"
]
},
"observables": {
"ips": [
"203.0.113.201"
],
"domains": [],
"urls": [],
"hashes": []
},
"evidence": [
"High-volume failed attempts over a short period.",
"Source IP attributed to a residential proxy provider.",
"No matching successful sign-in or MFA event found."
],
"patterns": [
"verdict:uncertain",
"scenario:o365_suspicious_login",
"alert_type:azuread_password_spray_attempt"
],
"related_refs": {
"playbooks": [
"PB-O365-LOGIN-001"
],
"kb": [
"KB-O365-IMPOSSIBLE-TRAVEL"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1004.json",
"tags": [
"o365",
"login",
"password-spray",
"pending"
]
}

55
evaluation/datasets/normalized_cases/CASE-2026-1005.json Normal file
View File

@ -0,0 +1,55 @@
{
"id": "CASE-2026-1005",
"memory_type": "case",
"scenario": "o365_suspicious_login",
"title": "Traveling executive triggered impossible travel but activity was legitimate",
"abstract": "Executive account triggered impossible travel due to corporate VPN exit node while the user was on an approved overseas trip.",
"verdict": "false_positive",
"severity": "medium",
"entities": {
"users": [
"grace@corp.example"
],
"hosts": [
"VIP-LAPTOP-01"
],
"mailboxes": [
"grace@corp.example"
]
},
"observables": {
"ips": [
"192.0.2.90",
"203.0.113.77"
],
"domains": [],
"urls": [],
"hashes": []
},
"evidence": [
"Approved travel request existed.",
"One login originated from corporate VPN exit node.",
"Device and user agent were consistent with known user profile."
],
"patterns": [
"verdict:false_positive",
"scenario:o365_suspicious_login",
"alert_type:azuread_impossible_travel"
],
"related_refs": {
"playbooks": [
"PB-O365-LOGIN-001"
],
"kb": [
"KB-O365-IMPOSSIBLE-TRAVEL"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_cases/o365_suspicious_login/CASE-2026-1005.json",
"tags": [
"o365",
"login",
"false-positive",
"travel"
]
}

34
evaluation/datasets/normalized_kb/KB-CRED-HARVEST-PATTERNS.json Normal file
View File

@ -0,0 +1,34 @@
{
"id": "KB-CRED-HARVEST-PATTERNS",
"memory_type": "knowledge",
"doc_type": "kb",
"scenario": "phishing",
"title": "Credential Harvesting Indicators",
"abstract": "Common indicators that a phishing case involves credential harvesting rather than simple spam or benign mail.",
"key_points": [
"Landing page mimics Microsoft 365 or common SaaS login pages.",
"HTML attachment often acts as a redirector rather than containing malware.",
"Credential harvest campaigns frequently reuse branding and lures across tenants."
],
"investigation_guidance": [
"Capture full redirect chain.",
"Look for post-click login anomalies in identity logs.",
"Search for same lure across multiple mailboxes."
],
"decision_points": [
"User click plus sign-in anomaly greatly increases confidence.",
"Branding reuse can help link separate phishing cases into one campaign."
],
"related_refs": {
"playbooks": [
"PB-PHISH-001"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_kb/kb/KB-CRED-HARVEST-PATTERNS.json",
"tags": [
"kb",
"phishing",
"credential-harvest"
]
}

34
evaluation/datasets/normalized_kb/KB-O365-IMPOSSIBLE-TRAVEL.json Normal file
View File

@ -0,0 +1,34 @@
{
"id": "KB-O365-IMPOSSIBLE-TRAVEL",
"memory_type": "knowledge",
"doc_type": "kb",
"scenario": "o365_suspicious_login",
"title": "Interpreting O365 Impossible Travel Alerts",
"abstract": "Guidance for validating impossible travel alerts, including VPN, proxy, and approved travel false-positive conditions.",
"key_points": [
"Impossible travel must be validated against user travel context.",
"VPN egress and cloud proxy routing are common false-positive sources.",
"Pair sign-in anomaly with MFA, mailbox, or device anomalies before concluding compromise."
],
"investigation_guidance": [
"Validate source ASN and IP history.",
"Check user-approved travel or remote work context.",
"Compare device ID and user agent consistency."
],
"decision_points": [
"User denial of travel plus new device strongly increases confidence.",
"Approved travel and trusted VPN topology reduce confidence."
],
"related_refs": {
"playbooks": [
"PB-O365-LOGIN-001"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_kb/kb/KB-O365-IMPOSSIBLE-TRAVEL.json",
"tags": [
"kb",
"o365",
"impossible-travel"
]
}

33
evaluation/datasets/normalized_kb/KB-O365-INBOX-RULE-ABUSE.json Normal file
View File

@ -0,0 +1,33 @@
{
"id": "KB-O365-INBOX-RULE-ABUSE",
"memory_type": "knowledge",
"doc_type": "kb",
"scenario": "o365_suspicious_login",
"title": "Inbox Rule Abuse After Account Compromise",
"abstract": "Common mailbox persistence behaviors after O365 account compromise, especially rule creation to hide or forward finance emails.",
"key_points": [
"Attackers often hide financial emails using move-to-folder rules.",
"Forwarding and delete rules are strong post-compromise indicators.",
"Mailbox audit logs should be reviewed immediately after suspicious login confirmation."
],
"investigation_guidance": [
"Enumerate all inbox rules and forwarding settings.",
"Check mailbox audit timeline around suspicious sign-in.",
"Review OAuth consents if inbox rules are absent but suspicious mail actions continue."
],
"decision_points": [
"Inbox rule creation shortly after suspicious login strongly supports compromise verdict."
],
"related_refs": {
"playbooks": [
"PB-O365-LOGIN-001"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_kb/kb/KB-O365-INBOX-RULE-ABUSE.json",
"tags": [
"kb",
"o365",
"inbox-rule"
]
}

33
evaluation/datasets/normalized_kb/KB-O365-MFA-FATIGUE.json Normal file
View File

@ -0,0 +1,33 @@
{
"id": "KB-O365-MFA-FATIGUE",
"memory_type": "knowledge",
"doc_type": "kb",
"scenario": "o365_suspicious_login",
"title": "MFA Fatigue Detection Notes",
"abstract": "Patterns for identifying MFA fatigue / push bombing during account compromise attempts.",
"key_points": [
"Repeated MFA prompts preceding one successful prompt is suspicious.",
"User-reported prompt fatigue is strong supporting evidence.",
"MFA fatigue is often coupled with credential theft rather than password spray alone."
],
"investigation_guidance": [
"Review MFA event counts and timing.",
"Check if the user acknowledged unexpected prompts.",
"Look for subsequent session hijacking or mailbox abuse."
],
"decision_points": [
"Prompt flood plus user denial usually warrants immediate containment."
],
"related_refs": {
"playbooks": [
"PB-O365-LOGIN-001"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_kb/kb/KB-O365-MFA-FATIGUE.json",
"tags": [
"kb",
"o365",
"mfa-fatigue"
]
}

34
evaluation/datasets/normalized_kb/KB-PHISH-HEADER-CHECK.json Normal file
View File

@ -0,0 +1,34 @@
{
"id": "KB-PHISH-HEADER-CHECK",
"memory_type": "knowledge",
"doc_type": "kb",
"scenario": "phishing",
"title": "Phishing Header Validation Checklist",
"abstract": "Checklist for validating sender identity, domain reputation, and authentication results in suspected phishing emails.",
"key_points": [
"Review SPF, DKIM, and DMARC alignment.",
"Compare display name, envelope sender, and reply-to anomalies.",
"Check domain age and known-good communication history."
],
"investigation_guidance": [
"Use message trace and header parser.",
"Compare sender domain with vendor allowlist.",
"Escalate lookalike domains even when content appears business-relevant."
],
"decision_points": [
"Newly observed domains with failed auth are high-risk.",
"Benign vendor mail often has consistent historical sending patterns."
],
"related_refs": {
"playbooks": [
"PB-PHISH-001"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_kb/kb/KB-PHISH-HEADER-CHECK.json",
"tags": [
"kb",
"phishing",
"email-header"
]
}

36
evaluation/datasets/normalized_kb/PB-O365-LOGIN-001.json Normal file
View File

@ -0,0 +1,36 @@
{
"id": "PB-O365-LOGIN-001",
"memory_type": "knowledge",
"doc_type": "playbook",
"scenario": "o365_suspicious_login",
"title": "O365 Suspicious Login Investigation Playbook",
"abstract": "Standard investigation steps for suspicious Entra ID sign-ins, impossible travel, MFA abuse, and follow-on mailbox abuse.",
"key_points": [
"Confirm user travel and business context.",
"Review sign-in logs, device IDs, and user agents.",
"Inspect downstream actions such as inbox rules, app consent, and forwarding."
],
"investigation_guidance": [
"Correlate MFA telemetry with sign-in sequence.",
"Check risky sign-ins and risky users views.",
"Revoke sessions and reset credentials when compromise is confirmed."
],
"decision_points": [
"Impossible travel alone is insufficient without corroborating evidence.",
"Inbox rule creation after foreign login strongly increases confidence of compromise."
],
"related_refs": {
"kb": [
"KB-O365-IMPOSSIBLE-TRAVEL",
"KB-O365-MFA-FATIGUE",
"KB-O365-INBOX-RULE-ABUSE"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_kb/playbooks/PB-O365-LOGIN-001.json",
"tags": [
"playbook",
"o365",
"login"
]
}

35
evaluation/datasets/normalized_kb/PB-PHISH-001.json Normal file
View File

@ -0,0 +1,35 @@
{
"id": "PB-PHISH-001",
"memory_type": "knowledge",
"doc_type": "playbook",
"scenario": "phishing",
"title": "Phishing Email Investigation Playbook",
"abstract": "Standard investigation steps for suspicious email, credential harvesting, and BEC-like cases.",
"key_points": [
"Validate sender authentication results.",
"Inspect landing URL and attachment behavior.",
"Check whether the user clicked or submitted credentials."
],
"investigation_guidance": [
"Query email telemetry for same sender, subject, or URL.",
"Review mailbox click logs and endpoint browser artifacts.",
"Reset credentials if submission is suspected."
],
"decision_points": [
"If sender auth fails and user interaction exists, treat as likely phishing.",
"If destination is allowlisted and communication pattern is expected, investigate false positive path."
],
"related_refs": {
"kb": [
"KB-PHISH-HEADER-CHECK",
"KB-CRED-HARVEST-PATTERNS"
],
"cases": []
},
"source_path": "/home/tom/soc_memory_poc/evaluation/datasets/mock_kb/playbooks/PB-PHISH-001.json",
"tags": [
"playbook",
"phishing",
"email"
]
}