From 1897c87fd035d882d9e66170f0cb7f99524bf6dd Mon Sep 17 00:00:00 2001 From: obsidianforensics Date: Mon, 12 Feb 2024 22:14:40 +0000 Subject: [PATCH 1/2] Update dfiq.org and add "Approach Glossary" page --- data/approaches/Q1036.10.yaml | 1 + data/approaches/Q1036.11.yaml | 1 + data/approaches/Q1037.10.yaml | 1 + data/approaches/Q1037.11.yaml | 1 + data/approaches/Q1037.12.yaml | 1 + dfiq.py | 66 ++++++++++ site/docs/contributing/approach_glossary.md | 134 ++++++++++++++++++++ site/docs/contributing/specification.md | 2 +- site/docs/questions/Q1001.md | 29 ++++- site/docs/questions/Q1018.md | 17 ++- site/docs/questions/Q1019.md | 2 +- site/docs/questions/Q1020.md | 11 +- site/docs/questions/Q1024.md | 2 +- site/docs/questions/Q1036.md | 20 ++- site/docs/questions/Q1037.md | 30 ++++- site/docs/questions/Q1074.md | 11 +- site/docs/questions/index.md | 40 +++--- templates/approach_glossary.jinja2 | 115 +++++++++++++++++ templates/question_with_approaches.jinja2 | 6 +- 19 files changed, 435 insertions(+), 55 deletions(-) create mode 100644 site/docs/contributing/approach_glossary.md create mode 100644 templates/approach_glossary.jinja2 diff --git a/data/approaches/Q1036.10.yaml b/data/approaches/Q1036.10.yaml index 974ae4a..d3c123a 100644 --- a/data/approaches/Q1036.10.yaml +++ b/data/approaches/Q1036.10.yaml @@ -29,6 +29,7 @@ description: references: - "[Prefetch on the ForensicsWiki](https://forensics.wiki/prefetch/)" - "[PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/)" + - "[Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc)" view: data: - type: ForensicArtifact diff --git a/data/approaches/Q1036.11.yaml b/data/approaches/Q1036.11.yaml index 5edb89f..7fc9c0e 100644 --- a/data/approaches/Q1036.11.yaml +++ b/data/approaches/Q1036.11.yaml @@ -29,6 +29,7 @@ description: references: - "[4688(S): A new process has been created](https://learn.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4688)" - "[PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/)" + - "[Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc)" view: data: - type: ForensicArtifact diff --git a/data/approaches/Q1037.10.yaml b/data/approaches/Q1037.10.yaml index 89784d1..d558239 100644 --- a/data/approaches/Q1037.10.yaml +++ b/data/approaches/Q1037.10.yaml @@ -29,6 +29,7 @@ description: references: - "[Prefetch on the ForensicsWiki](https://forensics.wiki/prefetch/)" - "[PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/)" + - "[Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc)" view: data: - type: ForensicArtifact diff --git a/data/approaches/Q1037.11.yaml b/data/approaches/Q1037.11.yaml index c603df5..009de3f 100644 --- a/data/approaches/Q1037.11.yaml +++ b/data/approaches/Q1037.11.yaml @@ -29,6 +29,7 @@ description: references: - "[4688(S): A new process has been created](https://learn.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4688)" - "[PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/)" + - "[Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc)" view: data: - type: ForensicArtifact diff --git a/data/approaches/Q1037.12.yaml b/data/approaches/Q1037.12.yaml index 9f3aa76..f923ce8 100644 --- a/data/approaches/Q1037.12.yaml +++ b/data/approaches/Q1037.12.yaml @@ -29,6 +29,7 @@ description: references: - "[4688(S): A new process has been created](https://learn.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4688)" - "[PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/)" + - "[Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc)" view: data: - type: ForensicArtifact diff --git a/dfiq.py b/dfiq.py index 29a019f..9873c0e 100644 --- a/dfiq.py +++ b/dfiq.py @@ -16,6 +16,7 @@ import logging import networkx as nx import os +import re import yamale import yaml @@ -556,3 +557,68 @@ def generate_question_index_md(self, allow_internal: bool = False) -> None: os.path.join(self.markdown_output_path, "questions", "index.md"), mode="w" ) as file: file.write(content) + + def generate_approach_glossary_md(self, allow_internal: bool = False) -> None: + """Generates Markdown for the Approach Glossary page, listing common items in Approaches. + + Args: + allow_internal (bool): Check if generating from internal items is allowed. + """ + data_type_and_value = {} + processor_and_analysis_names = {} + analysis_step_types = set() + step_variables = set() + for dfiq_id, component in self.components.items(): + if not isinstance(component, Approach): + continue + + if not allow_internal and component.is_internal: + continue + + for d in component.view.get('data'): + if not data_type_and_value.get(d['type']): + data_type_and_value[d['type']] = set() + data_type_and_value[d['type']].add(d['value']) + + for p in component.view.get('processors'): + if not processor_and_analysis_names.get(p['name']): + processor_and_analysis_names[p['name']] = set() + + for analysis in p['analysis']: + processor_and_analysis_names[p['name']].add(analysis['name']) + + for step in analysis['steps']: + analysis_step_types.add(step['type']) + m = re.findall(r'\{.*?\}', step['value']) + if m: + step_variables.update(m) + + if not self.markdown_output_path: + raise ValueError('Markdown output path not specified') + + descriptions = { + 'ForensicArtifact': 'This corresponds to the name of a ForensicArtifact, an existing repository of ' + 'machine-readable digital forensic artifacts (' + 'https://github.com/ForensicArtifacts/artifacts). Using this type is preferred when ' + 'the data is a host-based file/artifact, but other methods are available as well (if ' + 'there isn\'t an existing relevant ForensicArtifact).', + 'description': 'Text description of the data type. `description` is often using in conjunction with ' + 'another data type to provide more context. It can also be used alone, either as a ' + 'placeholder or when more robust, programmatic data types do not fit.', + } + + template = self.jinja_env.get_template('approach_glossary.jinja2') + context = { + 'data_type_and_value': data_type_and_value, + 'processor_and_analysis_names': processor_and_analysis_names, + 'analysis_step_types': analysis_step_types, + 'step_variables': step_variables, + 'descriptions': descriptions, + 'components': self.components + } + content = template.render(context) + with open(os.path.join( + self.markdown_output_path, + 'contributing', + 'approach_glossary.md'), mode='w') as file: + file.write(content) diff --git a/site/docs/contributing/approach_glossary.md b/site/docs/contributing/approach_glossary.md new file mode 100644 index 0000000..238de0d --- /dev/null +++ b/site/docs/contributing/approach_glossary.md @@ -0,0 +1,134 @@ +# Approach Glossary + +DFIQ's Approaches contain the important "how" to answer the Questions. Approaches are also the most complicated +part of DFIQ, due to the amount of structured information they contain. DFIQ has a detailed +[specification](https://dfiq.org/contributing/specification) that is a useful reference for +creating new Approaches. However, some parts of Approaches need user-defined values that are beyond the specification. +This page is a glossary of currently-used values, generated from the +[DFIQ YAML files](https://github.com/google/dfiq/tree/main/data). + +When writing new Approaches, check this glossary first to see if there's already an existing term that fits with what +you're trying to do. If not, you are free to create a new one, but trying to reuse existing terms first will increase +consistency throughout DFIQ. These concepts (data type, processors, analysis steps) also may not be straight-forward at +first; the hope is that seeing some common values (and the linked usages) will help make them more clear. + +## Data + +This section (`view.data`) can have multiple ways describing the data needed for this approach. They should be thought +of as complementary or as alternates to each other (they can be "OR"d together, they do not need to be "AND"d). +Each is specified by a pair of `type` and `value`. + +Example (from [Q1001.10](https://github.com/google/dfiq/blob/main/data/approaches/Q1001.10.yaml#L39)): + +``` +view: + data: + - type: ForensicArtifact + value: BrowserHistory +``` + +Below are the current values of `type`, along with the `value`s set for each. + + +#### CrowdStrike + +For `type: CrowdStrike`, current entries for `value`: + +- DnsRequest +- PlatformEvents +- ProcessRollup + +#### ForensicArtifact +**Description**: This corresponds to the name of a ForensicArtifact, an existing repository of machine-readable digital forensic artifacts (https://github.com/ForensicArtifacts/artifacts). Using this type is preferred when the data is a host-based file/artifact, but other methods are available as well (if there isn't an existing relevant ForensicArtifact). + +For `type: ForensicArtifact`, current entries for `value`: + +- BrowserHistory +- NTFSUSNJournal +- SantaLogs +- WindowsEventLogs +- WindowsPrefetchFiles +- WindowsXMLEventLogSysmon + +#### description +**Description**: Text description of the data type. `description` is often using in conjunction with another data type to provide more context. It can also be used alone, either as a placeholder or when more robust, programmatic data types do not fit. + +For `type: description`, current entries for `value`: + +- Collect local browser history artifacts. These are often in the form of SQLite databases and JSON files in multiple directories. +- Files used by the Windows Prefetch service. +- Santa logs stored on the local disk; they may also be centralized off-system, but this artifact does not include those. +- The NTFS $UsnJnrl file system metadata file. This ForensicArtifact definition does not include the $J alternate data stream, but many tools collect it anyway. +- Windows Event Log files + + +## Processors + +A processor is what takes the data collected and processes it in some way to produce structured data an investigator +reviews. Multiple processors can be defined, as there are often multiple programs capable of doing similar processing +(example: log2timeline, Magnet Axiom, and Hindsight can all process browser history artifacts and deliver similar +results). + +Example (from [Q1001.10](https://github.com/google/dfiq/blob/main/data/approaches/Q1001.10.yaml#L58)): + +``` + processors: + - name: Plaso +``` + +Below are the currently-defined processors: + +- Crowdstrike Investigate (UI) [🔎](https://github.com/google/dfiq/search?q="name:%20Crowdstrike%20Investigate%20%28UI%29"+language%3AYAML) +- Hindsight [🔎](https://github.com/google/dfiq/search?q="name:%20Hindsight"+language%3AYAML) +- Plaso [🔎](https://github.com/google/dfiq/search?q="name:%20Plaso"+language%3AYAML) +- Splunk [🔎](https://github.com/google/dfiq/search?q="name:%20Splunk"+language%3AYAML) + +## Analysis Steps + +Under each analysis method will be a sequence of one or more maps with keys `description`, `type`, and `value`. +If there is more than one map, they should be processed in sequence in the analysis method (if applicable). In this +way, we can describe multiple chained steps of analysis (with the `description` being a way to communicate to the user +what exactly each "step" is doing, enabling a "show-your-work"-type capability). + +Example (from [Q1001.10](https://github.com/google/dfiq/blob/main/data/approaches/Q1001.10.yaml#L63)): + +``` + analysis: + - name: OpenSearch + steps: + - description: &filter-desc Filter the results to just file downloads + type: opensearch-query + value: data_type:("chrome:history:file_downloaded" OR "safari:downloads:entry") + - name: Python Notebook + steps: + - description: *filter-desc + type: pandas + value: query('data_type in ("chrome:history:file_downloaded", "safari:downloads:entry")') +``` + +#### `type` + +The contents of the `description` and `value` fields will vary wildly with little repetition, depending on what the +analysis step is doing, but the step `type` should be one of a few common values. + +Below are the currently-defined values of `type`: + +- GUI [🔎](https://github.com/google/dfiq/search?q="type:%20GUI"+language%3AYAML) +- manual [🔎](https://github.com/google/dfiq/search?q="type:%20manual"+language%3AYAML) +- opensearch-query [🔎](https://github.com/google/dfiq/search?q="type:%20opensearch-query"+language%3AYAML) +- pandas [🔎](https://github.com/google/dfiq/search?q="type:%20pandas"+language%3AYAML) +- splunk-query [🔎](https://github.com/google/dfiq/search?q="type:%20splunk-query"+language%3AYAML) + +#### Variable Substitution in step `value` + +The step's `value` may benefit from some using a specific term to make the step more precise. Common examples of this +include adding time bounds and filtering down to a specific identifier (user name, host, FQDN, or PID, for example). + +DFIQ's convention for denoting a variable to be substituted when used is to wrap the term in **{ }**. + +==More standardization is needed here to define common variables (such as timestamps in a particular format).== + +Below are the currently-used variables in analysis steps: + +- {file_reference value} [🔎](https://github.com/google/dfiq/search?q="%7Bfile_reference%20value%7D"+language%3AYAML) +- {hostname} [🔎](https://github.com/google/dfiq/search?q="%7Bhostname%7D"+language%3AYAML) diff --git a/site/docs/contributing/specification.md b/site/docs/contributing/specification.md index 6a8c83c..8262007 100644 --- a/site/docs/contributing/specification.md +++ b/site/docs/contributing/specification.md @@ -32,7 +32,7 @@ A DFIQ document that conforms to the DFIQ Specification is represented in YAML. Both the 4- and 2-digit numbers start with a 1 (or higher) for components appropriate for external use. Numbers starting with a 0 are reserved for internal use (think of it like private IP address space). Users of DFIQ can use these IDs for their internal components without worrying about collisions with public components. -[DFIQ on GitHub](https://github.com/google/dfiq) will serve as the "definitive" central repository to manage +[DFIQ on GitHub](https://github.com/google/dfiq) will serve as the definitive central repository to manage public DFIQ components (and their IDs). ## Schema diff --git a/site/docs/questions/Q1001.md b/site/docs/questions/Q1001.md index 4231882..b33f351 100644 --- a/site/docs/questions/Q1001.md +++ b/site/docs/questions/Q1001.md @@ -77,7 +77,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Collect local browser history artifacts. These are often in the form of SQLite databases and JSON files in multiple directories. - - ForensicArtifact: BrowserHistory + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: BrowserHistory ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=BrowserHistory)) ### ⚙️ Processors @@ -94,7 +99,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: @@ -128,7 +133,7 @@ relevant configuration options are. df.query('data_type in ("chrome:history:file_downloaded", "safari:downloads:entry")') ``` === "Hindsight" - More information on [Hindsight](https://forensics.wiki/Hindsight). + More information on [Hindsight](https://forensics.wiki/hindsight). Recommended options: @@ -208,7 +213,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Santa logs stored on the local disk; they may also be centralized off-system, but this artifact does not include those. - - ForensicArtifact: SantaLogs + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: SantaLogs ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=SantaLogs)) ### ⚙️ Processors @@ -225,7 +235,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: @@ -298,7 +308,12 @@ The following data source(s) are needed for this approach to the question. **Description** : The NTFS $UsnJnrl file system metadata file. This ForensicArtifact definition does not include the $J alternate data stream, but many tools collect it anyway. - - ForensicArtifact: NTFSUSNJournal + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: NTFSUSNJournal ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=NTFSUSNJournal)) ### ⚙️ Processors @@ -315,7 +330,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: diff --git a/site/docs/questions/Q1018.md b/site/docs/questions/Q1018.md index a683e0d..4594a64 100644 --- a/site/docs/questions/Q1018.md +++ b/site/docs/questions/Q1018.md @@ -23,7 +23,7 @@ hide: - [Use Crowdstrike event search to link source processes to DNS queries [Q1018.11]](#use-crowdstrike-event-search-to-link-source-processes-to-dns-queries) - CrowdStrike records the source process ID (ContextProcessId) for DNSRequest event. - Tags: CrowdStrike DNS -- [Use Sysmon (Event ID 22) to link source processes to DNS queries [Q1018.12]](#use-sysmon-(event-id-22)-to-link-source-processes-to-dns-queries) +- [Use Sysmon (Event ID 22) to link source processes to DNS queries [Q1018.12]](#use-sysmon-event-id-22-to-link-source-processes-to-dns-queries) - Sysmon Event ID 22 DnsQuery stores source process ID - Tags: Sysmon DNS Windows @@ -81,7 +81,7 @@ relevant configuration options are. === "Crowdstrike Investigate (Ui)" - More information on [Crowdstrike Investigate (Ui)](https://forensics.wiki/Crowdstrike Investigate (UI)). + More information on [Crowdstrike Investigate (Ui)](https://forensics.wiki/crowdstrike investigate (ui)). #### 📊 Analysis @@ -145,7 +145,7 @@ relevant configuration options are. === "Splunk" - More information on [Splunk](https://forensics.wiki/Splunk). + More information on [Splunk](https://forensics.wiki/splunk). #### 📊 Analysis @@ -191,7 +191,12 @@ ambiguity. The following data source(s) are needed for this approach to the question. - - ForensicArtifact: WindowsXMLEventLogSysmon + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: WindowsXMLEventLogSysmon ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=WindowsXMLEventLogSysmon)) ### ⚙️ Processors @@ -208,7 +213,7 @@ relevant configuration options are. === "Splunk" - More information on [Splunk](https://forensics.wiki/Splunk). + More information on [Splunk](https://forensics.wiki/splunk). #### 📊 Analysis @@ -226,7 +231,7 @@ relevant configuration options are. - Value: source="xmlwineventlog:microsoft-windows-sysmon/operational" EventCode=22 | table _time, host, process_id, process_path === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). #### 📊 Analysis diff --git a/site/docs/questions/Q1019.md b/site/docs/questions/Q1019.md index 45b346c..eed6430 100644 --- a/site/docs/questions/Q1019.md +++ b/site/docs/questions/Q1019.md @@ -76,7 +76,7 @@ relevant configuration options are. === "Splunk" - More information on [Splunk](https://forensics.wiki/Splunk). + More information on [Splunk](https://forensics.wiki/splunk). #### 📊 Analysis diff --git a/site/docs/questions/Q1020.md b/site/docs/questions/Q1020.md index 4e925a8..1209086 100644 --- a/site/docs/questions/Q1020.md +++ b/site/docs/questions/Q1020.md @@ -70,7 +70,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Collect local browser history artifacts. These are often in the form of SQLite databases and JSON files in multiple directories. - - ForensicArtifact: BrowserHistory + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: BrowserHistory ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=BrowserHistory)) ### ⚙️ Processors @@ -87,7 +92,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: @@ -121,7 +126,7 @@ relevant configuration options are. df.query('data_type in ("chrome:history:page_visited", "firefox:places:page_visited", "safari:history:visit_sqlite")') ``` === "Hindsight" - More information on [Hindsight](https://forensics.wiki/Hindsight). + More information on [Hindsight](https://forensics.wiki/hindsight). Recommended options: diff --git a/site/docs/questions/Q1024.md b/site/docs/questions/Q1024.md index 8b0c300..814c6e5 100644 --- a/site/docs/questions/Q1024.md +++ b/site/docs/questions/Q1024.md @@ -75,7 +75,7 @@ relevant configuration options are. === "Splunk" - More information on [Splunk](https://forensics.wiki/Splunk). + More information on [Splunk](https://forensics.wiki/splunk). #### 📊 Analysis diff --git a/site/docs/questions/Q1036.md b/site/docs/questions/Q1036.md index af304d1..1dda94f 100644 --- a/site/docs/questions/Q1036.md +++ b/site/docs/questions/Q1036.md @@ -39,6 +39,7 @@ Windows Prefetch files are designed to speed up application start times. To do t #### References - [Prefetch on the ForensicsWiki](https://forensics.wiki/prefetch/) - [PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/) + - [Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc) ### 📝 Notes @@ -67,7 +68,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Files used by the Windows Prefetch service. - - ForensicArtifact: WindowsPrefetchFiles + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: WindowsPrefetchFiles ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=WindowsPrefetchFiles)) ### ⚙️ Processors @@ -84,7 +90,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: @@ -115,6 +121,7 @@ Windows systems can be set to log new process creation events to the Security ev #### References - [4688(S): A new process has been created](https://learn.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4688) - [PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/) + - [Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc) ### 📝 Notes @@ -141,7 +148,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Windows Event Log files - - ForensicArtifact: WindowsEventLogs + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: WindowsEventLogs ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=WindowsEventLogs)) ### ⚙️ Processors @@ -158,7 +170,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: diff --git a/site/docs/questions/Q1037.md b/site/docs/questions/Q1037.md index 9d4953b..a68913c 100644 --- a/site/docs/questions/Q1037.md +++ b/site/docs/questions/Q1037.md @@ -42,6 +42,7 @@ Windows Prefetch files are designed to speed up application start times. To do t #### References - [Prefetch on the ForensicsWiki](https://forensics.wiki/prefetch/) - [PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/) + - [Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc) ### 📝 Notes @@ -70,7 +71,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Files used by the Windows Prefetch service. - - ForensicArtifact: WindowsPrefetchFiles + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: WindowsPrefetchFiles ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=WindowsPrefetchFiles)) ### ⚙️ Processors @@ -87,7 +93,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: @@ -118,6 +124,7 @@ Windows systems can be set to log new process creation events to the Security ev #### References - [4688(S): A new process has been created](https://learn.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4688) - [PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/) + - [Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc) ### 📝 Notes @@ -144,7 +151,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Windows Event Log files - - ForensicArtifact: WindowsEventLogs + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: WindowsEventLogs ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=WindowsEventLogs)) ### ⚙️ Processors @@ -161,7 +173,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: @@ -192,6 +204,7 @@ Windows systems can be set to log new service installations to the System event #### References - [4688(S): A new process has been created](https://learn.microsoft.com/en-us/windows/security/threat-protection/auditing/event-4688) - [PsExec on MITRE ATT&CK](https://attack.mitre.org/software/S0029/) + - [Detecting PsExec Usage by 13Cubed](https://www.youtube.com/watch?v=oVM1nQhDZQc) ### 📝 Notes @@ -218,7 +231,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Windows Event Log files - - ForensicArtifact: WindowsEventLogs + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: WindowsEventLogs ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=WindowsEventLogs)) ### ⚙️ Processors @@ -235,7 +253,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: diff --git a/site/docs/questions/Q1074.md b/site/docs/questions/Q1074.md index 4023e1a..53375e6 100644 --- a/site/docs/questions/Q1074.md +++ b/site/docs/questions/Q1074.md @@ -82,7 +82,7 @@ relevant configuration options are. === "Splunk" - More information on [Splunk](https://forensics.wiki/Splunk). + More information on [Splunk](https://forensics.wiki/splunk). #### 📊 Analysis @@ -132,7 +132,12 @@ The following data source(s) are needed for this approach to the question. **Description** : Windows Event Log files - - ForensicArtifact: WindowsEventLogs + +**Type** +: [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) + +**Value** +: WindowsEventLogs ([view on GitHub](https://github.com/ForensicArtifacts/artifacts/search?q=WindowsEventLogs)) ### ⚙️ Processors @@ -149,7 +154,7 @@ relevant configuration options are. === "Plaso" - More information on [Plaso](https://forensics.wiki/Plaso). + More information on [Plaso](https://forensics.wiki/plaso). Recommended options: diff --git a/site/docs/questions/index.md b/site/docs/questions/index.md index 4fd3674..c51836f 100644 --- a/site/docs/questions/index.md +++ b/site/docs/questions/index.md @@ -24,7 +24,7 @@ Some quick facts about Questions in DFIQ: | ID | Question | Tags | Approaches | | ----------- | ------------------------------------ | ---- | ---------- | -| Q1001 | What files were downloaded using a web browser? | USN Journal Safari NTFS Edge Web Browser SQLite Chrome Windows macOS | 3 | +| Q1001 | What files were downloaded using a web browser? | Windows Edge USN Journal Safari macOS SQLite Chrome NTFS Web Browser | 3 | | Q1002 | What USB devices were attached to a computer? | | 0 | | Q1003 | What files were copied from a computer to a USB device? | | 0 | | Q1004 | What screenshots were taken on a computer? | | 0 | @@ -41,13 +41,13 @@ Some quick facts about Questions in DFIQ: | Q1015 | What syncing activities did external "cloud storage" applications do on a computer? | | 0 | | Q1016 | What files were sent externally via email attachments? | | 0 | | Q1017 | Were any files collected into a container? | | 0 | -| Q1018 | What process made the DNS query? | DNS CrowdStrike Windows Sysmon | 3 | -| Q1019 | What web browsers were running at a given time? | Web Browser CrowdStrike Process Execution | 1 | -| Q1020 | What pages did web browsers visit? | Internet Explorer Safari Web Browser SQLite Firefox Chrome Edge | 1 | +| Q1018 | What process made the DNS query? | CrowdStrike Sysmon DNS Windows | 3 | +| Q1019 | What web browsers were running at a given time? | CrowdStrike Process Execution Web Browser | 1 | +| Q1020 | What pages did web browsers visit? | SQLite Edge Chrome Firefox Internet Explorer Safari Web Browser | 1 | | Q1021 | What Chrome extensions are installed? | Web Browser | 0 | -| Q1022 | What actions did a Chrome extension perform? | Chrome Web Browser Browser Extension | 0 | +| Q1022 | What actions did a Chrome extension perform? | Chrome Browser Extension Web Browser | 0 | | Q1023 | Is a given Chrome extension associated with a given domain? | Chrome T1176 | 0 | -| Q1024 | Was an Incognito/Private browser session used? | Web Browser CrowdStrike Process Execution | 1 | +| Q1024 | Was an Incognito/Private browser session used? | CrowdStrike Process Execution Web Browser | 1 | | Q1025 | What external accounts has the actor used in their web browser? | | 0 | | Q1026 | What files were downloaded from messaging apps? | | 0 | | Q1027 | Are there any sudden changes in the number of files on a device? | | 0 | @@ -59,8 +59,8 @@ Some quick facts about Questions in DFIQ: | Q1033 | Was shell history cleared? | | 0 | | Q1034 | Was content copied from a file? | | 0 | | Q1035 | Were any copies made of sensitive files? | | 0 | -| Q1036 | Have there been any executions of PsExec? | Prefetch Windows Event Logs | 2 | -| Q1037 | Have there been any executions of PsExeSrv? | Prefetch Windows Event Logs | 3 | +| Q1036 | Have there been any executions of PsExec? | Prefetch Event Logs Windows | 2 | +| Q1037 | Have there been any executions of PsExeSrv? | Prefetch Event Logs Windows | 3 | | Q1038 | Were any files sent via CLI utilities? | | 0 | | Q1039 | Are there any detections of exposed service account credentials? | | 0 | | Q1040 | Are there any interaction with the cloud project resources from an unknown external IP address (including Tor Exit nodes, C2 tagged addresses)? | | 0 | @@ -76,28 +76,28 @@ Some quick facts about Questions in DFIQ: | Q1050 | Are there any GCS buckets within the cloud project shared externally? | | 0 | | Q1051 | Are there any signs of data within the cloud project being transferred externally? | | 0 | | Q1052 | What browser extensions (non-Chrome) are installed? | Web Browser | 0 | -| Q1053 | What Launch Agents are configured? | macOS T1543.001 | 0 | -| Q1054 | What Launch Daemons are configured? | T1543.004 macOS | 0 | -| Q1055 | What Windows logon scripts are configured? | Windows T1037.001 | 0 | -| Q1056 | What login hooks are configured? | macOS T1037.002 | 0 | -| Q1057 | What network logon scripts are configured? | T1037.003 Windows | 0 | -| Q1058 | What domain accounts have been created? | Windows T1136.002 | 0 | -| Q1059 | What local accounts have been created? | macOS Windows T1136.001 Linux | 0 | +| Q1053 | What Launch Agents are configured? | T1543.001 macOS | 0 | +| Q1054 | What Launch Daemons are configured? | macOS T1543.004 | 0 | +| Q1055 | What Windows logon scripts are configured? | T1037.001 Windows | 0 | +| Q1056 | What login hooks are configured? | T1037.002 macOS | 0 | +| Q1057 | What network logon scripts are configured? | Windows T1037.003 | 0 | +| Q1058 | What domain accounts have been created? | T1136.002 Windows | 0 | +| Q1059 | What local accounts have been created? | T1136.001 macOS Linux Windows | 0 | | Q1060 | What AT jobs are configured? | macOS Linux T1053.002 | 0 | -| Q1061 | What Scheduled Tasks are configured? | Windows T1053.005 | 0 | +| Q1061 | What Scheduled Tasks are configured? | T1053.005 Windows | 0 | | Q1062 | What cron jobs are configured? | macOS Linux T1053.003 | 0 | | Q1063 | What periodic scripts are configured? | macOS | 0 | | Q1064 | What systemd timers are configured? | T1053.006 Linux | 0 | | Q1065 | What files are referenced in Registry "Run" keys? | T1547.001 Windows | 0 | | Q1066 | What items are in startup folders? | T1547.001 Windows | 0 | -| Q1067 | What system services are installed? | Windows macOS Linux | 0 | -| Q1068 | When were system services last modified? | Windows macOS Linux | 0 | +| Q1067 | What system services are installed? | macOS Linux Windows | 0 | +| Q1068 | When were system services last modified? | macOS Linux Windows | 0 | | Q1069 | Are there any indications of dylib hijacking? | macOS T1574.004 | 0 | | Q1070 | Are there any indications of dylib proxying? | macOS | 0 | | Q1071 | Are there any indications of dynamic linker hijacking? | T1574.006 macOS Linux | 0 | -| Q1072 | Did Chrome's DNS Prefetching cause a DNS query? | Web Browser DNS | 0 | +| Q1072 | Did Chrome's DNS Prefetching cause a DNS query? | DNS Web Browser | 0 | | Q1073 | Have there been any modifications to the "hosts" file? | | 0 | -| Q1074 | Were any system event logs cleared? | CrowdStrike macOS Windows T1070.002 Linux Event Logs T1070.001 | 2 | +| Q1074 | Were any system event logs cleared? | CrowdStrike T1070.002 Windows T1070.001 Event Logs macOS Linux | 2 | | Q1075 | Is the recipient account controlled by the sender? | | 0 | | Q1076 | Were the files screenshots? | | 0 | | Q1077 | Were there bulk downloads from Google Drive? | | 0 | diff --git a/templates/approach_glossary.jinja2 b/templates/approach_glossary.jinja2 new file mode 100644 index 0000000..2b1c676 --- /dev/null +++ b/templates/approach_glossary.jinja2 @@ -0,0 +1,115 @@ +{# templates/approach_glossary #} +# Approach Glossary + +DFIQ's Approaches contain the important "how" to answer the Questions. Approaches are also the most complicated +part of DFIQ, due to the amount of structured information they contain. DFIQ has a detailed +[specification](https://dfiq.org/contributing/specification) that is a useful reference for +creating new Approaches. However, some parts of Approaches need user-defined values that are beyond the specification. +This page is a glossary of currently-used values, generated from the +[DFIQ YAML files](https://github.com/google/dfiq/tree/main/data). + +When writing new Approaches, check this glossary first to see if there's already an existing term that fits with what +you're trying to do. If not, you are free to create a new one, but trying to reuse existing terms first will increase +consistency throughout DFIQ. These concepts (data type, processors, analysis steps) also may not be straight-forward at +first; the hope is that seeing some common values (and the linked usages) will help make them more clear. + +## Data + +This section (`view.data`) can have multiple ways describing the data needed for this approach. They should be thought +of as complementary or as alternates to each other (they can be "OR"d together, they do not need to be "AND"d). +Each is specified by a pair of `type` and `value`. + +Example (from [Q1001.10](https://github.com/google/dfiq/blob/main/data/approaches/Q1001.10.yaml#L39)): + +``` +view: + data: + - type: ForensicArtifact + value: BrowserHistory +``` + +Below are the current values of `type`, along with the `value`s set for each. + + +{% for data_type, values in data_type_and_value.items() | sort %} +#### {{ data_type }} +{% if descriptions.get(data_type) %} +**Description**: {{ descriptions.get(data_type) }} +{% endif %} + +For `type: {{ data_type }}`, current entries for `value`: + +{% for value in values | sort %} +- {{ value }} +{% endfor %} + +{% endfor %} + +## Processors + +A processor is what takes the data collected and processes it in some way to produce structured data an investigator +reviews. Multiple processors can be defined, as there are often multiple programs capable of doing similar processing +(example: log2timeline, Magnet Axiom, and Hindsight can all process browser history artifacts and deliver similar +results). + +Example (from [Q1001.10](https://github.com/google/dfiq/blob/main/data/approaches/Q1001.10.yaml#L58)): + +``` + processors: + - name: Plaso +``` + +Below are the currently-defined processors: + +{% for processor in processor_and_analysis_names | sort %} +- {{ processor }} [🔎](https://github.com/google/dfiq/search?q="name:%20{{ processor | urlencode }}"+language%3AYAML) +{% endfor %} + +## Analysis Steps + +Under each analysis method will be a sequence of one or more maps with keys `description`, `type`, and `value`. +If there is more than one map, they should be processed in sequence in the analysis method (if applicable). In this +way, we can describe multiple chained steps of analysis (with the `description` being a way to communicate to the user +what exactly each "step" is doing, enabling a "show-your-work"-type capability). + +Example (from [Q1001.10](https://github.com/google/dfiq/blob/main/data/approaches/Q1001.10.yaml#L63)): + +``` + analysis: + - name: OpenSearch + steps: + - description: &filter-desc Filter the results to just file downloads + type: opensearch-query + value: data_type:("chrome:history:file_downloaded" OR "safari:downloads:entry") + - name: Python Notebook + steps: + - description: *filter-desc + type: pandas + value: query('data_type in ("chrome:history:file_downloaded", "safari:downloads:entry")') +``` + +#### `type` + +The contents of the `description` and `value` fields will vary wildly with little repetition, depending on what the +analysis step is doing, but the step `type` should be one of a few common values. + +Below are the currently-defined values of `type`: + +{% for step in analysis_step_types | sort %} +- {{ step }} [🔎](https://github.com/google/dfiq/search?q="type:%20{{ step | urlencode }}"+language%3AYAML) +{% endfor %} + +#### Variable Substitution in step `value` + +The step's `value` may benefit from some using a specific term to make the step more precise. Common examples of this +include adding time bounds and filtering down to a specific identifier (user name, host, FQDN, or PID, for example). + +DFIQ's convention for denoting a variable to be substituted when used is to wrap the term in **{ }**. + +==More standardization is needed here to define common variables (such as timestamps in a particular format).== + +Below are the currently-used variables in analysis steps: + +{% for variable in step_variables | sort %} +- {{ variable }} [🔎](https://github.com/google/dfiq/search?q="{{ variable | urlencode }}"+language%3AYAML) +{% endfor %} \ No newline at end of file diff --git a/templates/question_with_approaches.jinja2 b/templates/question_with_approaches.jinja2 index a870d47..dc052f0 100644 --- a/templates/question_with_approaches.jinja2 +++ b/templates/question_with_approaches.jinja2 @@ -24,7 +24,7 @@ hide: {% for approach in question.approaches %} {% if components[approach].is_internal == False or allow_internal == True %} -- [{{ components[approach].name }} [{{ approach }}]](#{{ components[approach].name.replace(' ','-').lower() }}) +- [{{ components[approach].name }} [{{ approach }}]](#{{ components[approach].name.replace(' ','-').replace('(','').replace(')','').lower() }}) - {{ components[approach]['description']['summary'] }} - Tags: {{ components[approach]['tags']|join(' ') }} {% endif %} @@ -106,7 +106,7 @@ The following data source(s) are needed for this approach to the question. ```googlesql {{ data['value'] }} ``` -{% elif data['type'] == 'artifact' %} +{% elif data['type'] == 'ForensicArtifact' %} **Type** : [ForensicArtifact](https://github.com/ForensicArtifacts/artifacts#digital-forensics-artifacts-repository) @@ -135,7 +135,7 @@ relevant configuration options are. {% for processor in approach.view['processors'] %} === "{{ processor.name.title() }}" - More information on [{{ processor['name'].title() }}](https://forensics.wiki/{{ processor.name }}). + More information on [{{ processor['name'].title() }}](https://forensics.wiki/{{ processor.name|lower }}). {% if processor.get('options') %} Recommended options: From dc03b4df4243d3e2fa02e1ee2a2f17dc1152eba6 Mon Sep 17 00:00:00 2001 From: obsidianforensics Date: Mon, 12 Feb 2024 22:17:43 +0000 Subject: [PATCH 2/2] Code format with black --- dfiq.py | 66 ++++++++++++++++--------------- scripts/generate_site_markdown.py | 1 + 2 files changed, 35 insertions(+), 32 deletions(-) diff --git a/dfiq.py b/dfiq.py index 9873c0e..4eaa868 100644 --- a/dfiq.py +++ b/dfiq.py @@ -575,50 +575,52 @@ def generate_approach_glossary_md(self, allow_internal: bool = False) -> None: if not allow_internal and component.is_internal: continue - for d in component.view.get('data'): - if not data_type_and_value.get(d['type']): - data_type_and_value[d['type']] = set() - data_type_and_value[d['type']].add(d['value']) + for d in component.view.get("data"): + if not data_type_and_value.get(d["type"]): + data_type_and_value[d["type"]] = set() + data_type_and_value[d["type"]].add(d["value"]) - for p in component.view.get('processors'): - if not processor_and_analysis_names.get(p['name']): - processor_and_analysis_names[p['name']] = set() + for p in component.view.get("processors"): + if not processor_and_analysis_names.get(p["name"]): + processor_and_analysis_names[p["name"]] = set() - for analysis in p['analysis']: - processor_and_analysis_names[p['name']].add(analysis['name']) + for analysis in p["analysis"]: + processor_and_analysis_names[p["name"]].add(analysis["name"]) - for step in analysis['steps']: - analysis_step_types.add(step['type']) - m = re.findall(r'\{.*?\}', step['value']) + for step in analysis["steps"]: + analysis_step_types.add(step["type"]) + m = re.findall(r"\{.*?\}", step["value"]) if m: step_variables.update(m) if not self.markdown_output_path: - raise ValueError('Markdown output path not specified') + raise ValueError("Markdown output path not specified") descriptions = { - 'ForensicArtifact': 'This corresponds to the name of a ForensicArtifact, an existing repository of ' - 'machine-readable digital forensic artifacts (' - 'https://github.com/ForensicArtifacts/artifacts). Using this type is preferred when ' - 'the data is a host-based file/artifact, but other methods are available as well (if ' - 'there isn\'t an existing relevant ForensicArtifact).', - 'description': 'Text description of the data type. `description` is often using in conjunction with ' - 'another data type to provide more context. It can also be used alone, either as a ' - 'placeholder or when more robust, programmatic data types do not fit.', + "ForensicArtifact": "This corresponds to the name of a ForensicArtifact, an existing repository of " + "machine-readable digital forensic artifacts (" + "https://github.com/ForensicArtifacts/artifacts). Using this type is preferred when " + "the data is a host-based file/artifact, but other methods are available as well (if " + "there isn't an existing relevant ForensicArtifact).", + "description": "Text description of the data type. `description` is often using in conjunction with " + "another data type to provide more context. It can also be used alone, either as a " + "placeholder or when more robust, programmatic data types do not fit.", } - template = self.jinja_env.get_template('approach_glossary.jinja2') + template = self.jinja_env.get_template("approach_glossary.jinja2") context = { - 'data_type_and_value': data_type_and_value, - 'processor_and_analysis_names': processor_and_analysis_names, - 'analysis_step_types': analysis_step_types, - 'step_variables': step_variables, - 'descriptions': descriptions, - 'components': self.components + "data_type_and_value": data_type_and_value, + "processor_and_analysis_names": processor_and_analysis_names, + "analysis_step_types": analysis_step_types, + "step_variables": step_variables, + "descriptions": descriptions, + "components": self.components, } content = template.render(context) - with open(os.path.join( - self.markdown_output_path, - 'contributing', - 'approach_glossary.md'), mode='w') as file: + with open( + os.path.join( + self.markdown_output_path, "contributing", "approach_glossary.md" + ), + mode="w", + ) as file: file.write(content) diff --git a/scripts/generate_site_markdown.py b/scripts/generate_site_markdown.py index 1622393..d008e7f 100644 --- a/scripts/generate_site_markdown.py +++ b/scripts/generate_site_markdown.py @@ -23,3 +23,4 @@ dfiq_instance.generate_question_md(question.id) dfiq_instance.generate_question_index_md() +dfiq_instance.generate_approach_glossary_md()