From ee7ab07a5eb4e2a62350e437d6f7e89641547997 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 23 Mar 2024 21:46:14 +0900 Subject: [PATCH 1/9] feat: adjusted directory input in timeline option #133 --- src/takajopkg/automagic.nim | 136 ++++-- src/takajopkg/extractScriptblocks.nim | 88 ++-- src/takajopkg/general.nim | 136 +++--- src/takajopkg/listDomains.nim | 70 +-- src/takajopkg/listHashes.nim | 175 ++++---- src/takajopkg/listIpAddresses.nim | 44 +- src/takajopkg/listUndetectedEvtxFiles.nim | 8 +- src/takajopkg/listUnusedRules.nim | 7 +- src/takajopkg/splitCsvTimeline.nim | 119 ++--- src/takajopkg/splitJsonTimeline.nim | 71 +-- src/takajopkg/stackCmdlines.nim | 36 +- src/takajopkg/stackComputers.nim | 40 +- src/takajopkg/stackDNS.nim | 39 +- src/takajopkg/stackIpAddresses.nim | 39 +- src/takajopkg/stackLogons.nim | 122 +++--- src/takajopkg/stackProcesses.nim | 34 +- src/takajopkg/stackServices.nim | 43 +- src/takajopkg/stackTasks.nim | 69 +-- src/takajopkg/stackUsers.nim | 77 ++-- src/takajopkg/sysmonProcessTree.nim | 77 ++-- src/takajopkg/timelineLogon.nim | 152 ++++--- src/takajopkg/timelinePartitionDiagnostic.nim | 175 ++++---- src/takajopkg/timelineSuspiciousProcesses.nim | 407 +++++++++--------- src/takajopkg/timelineTasks.nim | 186 ++++---- src/takajopkg/ttpResult.nim | 115 ++--- src/takajopkg/ttpSummary.nim | 69 +-- src/takajopkg/ttpVisualize.nim | 46 +- 27 files changed, 1418 insertions(+), 1162 deletions(-) diff --git a/src/takajopkg/automagic.nim b/src/takajopkg/automagic.nim index fc2c0ca9..03c9a3a1 100644 --- a/src/takajopkg/automagic.nim +++ b/src/takajopkg/automagic.nim @@ -2,97 +2,147 @@ const AutoMagicMsg = "Automatically executes as many commands as possible and ou type AutoMagicCmd* = ref object of AbstractCmd - level: string - -proc autoMagic(level: string = "low", skipProgressBar:bool = false, displayTable:bool = false, output: string = "case-1", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) - if dirExists(output): - echo "The directory '" & output & "' exists. Please specify a new folder name." - quit(1) - else: - createDir(output) - + level: string + +proc autoMagic(level: string = "low", skipProgressBar: bool = false, + displayTable: bool = false, output: string = "case-1", quiet: bool = false, + timeline: string) = + checkArgs(quiet, timeline, level) + if dirExists(output): + echo "The directory '" & output & "' exists. Please specify a new folder name." + quit(1) + else: + createDir(output) + + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: # extract-scriptblocks -t ../hayabusa/timeline.jsonl -l -o case-1/scriptblock-logs/ - let cmd1 = ExtractScriptBlocksCmd(name:"extract-scriptblocks", level: level, skipProgressBar: skipProgressBar, displayTable:displayTable, timeline: timeline, output: output & "/scriptblock-logs") + let cmd1 = ExtractScriptBlocksCmd(name: "extract-scriptblocks", + level: level, skipProgressBar: skipProgressBar, + displayTable: displayTable, + timeline: timelinePath, output: output & "/scriptblock-logs") if not dirExists(output & "/scriptblock-logs/"): - createDir(output & "/scriptblock-logs/") + createDir(output & "/scriptblock-logs/") # list-domains -t ../hayabusa/timeline.jsonl -o case-1/ListDomains.txt - let cmd2 = ListDomainsCmd(name:"list-domains", displayTable:displayTable, timeline: timeline, output: output & "/ListDomains.txt") + let cmd2 = ListDomainsCmd(name: "list-domains", displayTable: displayTable, + timeline: timelinePath, output: output & "/ListDomains.txt") # list-domains -t ../hayabusa/timeline.jsonl -d -w -o case-1/ListDomains-Detailed.txt - let cmd3 = ListDomainsCmd(name:"list-domains(detailed)", displayTable:displayTable, timeline:timeline, output:output & "/ListDomainsDetailed.txt", includeSubdomains:true, includeWorkstations:true) + let cmd3 = ListDomainsCmd(name: "list-domains(detailed)", + displayTable: displayTable, timeline: timelinePath, output: output & + "/ListDomainsDetailed.txt", includeSubdomains: true, + includeWorkstations: true) # list-hashes -t ../hayabusa/timeline.jsonl -l -o case-1/hashes - let cmd4 = ListHashesCmd(name:"list-hashes", displayTable:displayTable, timeline:timeline, level:level, output:output & "/ListHashes") + let cmd4 = ListHashesCmd(name: "list-hashes", displayTable: displayTable, + timeline: timelinePath, level: level, output: output & "/ListHashes") # list-ip-addresses -t ../hayabusa/timeline.jsonl -o case-1/IP-Addresses.txt - let cmd5 = ListIpAddressesCmd(name:"list-ip-addresses", displayTable:displayTable, timeline:timeline, output:output & "/ListIP-Addresses.txt") + let cmd5 = ListIpAddressesCmd(name: "list-ip-addresses", + displayTable: displayTable, timeline: timelinePath, output: output & "/ListIP-Addresses.txt") # stack-cmdlines -t ../hayabusa/timeline.jsonl --level -o case-1/cmdlines.csv - let cmd6 = StackCmdlineCmd(name:"stack-cmdlines", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackCmdlines.csv") + let cmd6 = StackCmdlineCmd(name: "stack-cmdlines", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & "/StackCmdlines.csv") # stack-computers -t ../hayabusa/timeline.jsonl --level -o case-1/TargetComputers.csv - let cmd7 = StackComputersCmd(name: "stack-computers", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackTargetComputers.csv") + let cmd7 = StackComputersCmd(name: "stack-computers", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & "/StackTargetComputers.csv") # stack-computers -t ../hayabusa/timeline.jsonl --level --sourceComputers -o case-1/SourceComputers.csv - let cmd8 = StackComputersCmd(name: "stack-computers", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackSourceComputers.csv", sourceComputers:true) + let cmd8 = StackComputersCmd(name: "stack-computers", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & + "/StackSourceComputers.csv", sourceComputers: true) # stack-dns -t ../hayabusa/timeline.jsonl --level -o case-1/DNS.csv - let cmd9 = StackDNSCmd(name:"stack-dns", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackDNS.csv") + let cmd9 = StackDNSCmd(name: "stack-dns", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & "/StackDNS.csv") # stack-ip-addresses -t ../hayabusa/timeline.jsonl --level -o case-1/SourceIP-Addresses.csv - let cmd10 = StackIpAddressesCmd(name: "stack-ip-addresses(source)", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackSourceIP-Addresses.csv") + let cmd10 = StackIpAddressesCmd(name: "stack-ip-addresses(source)", + level: level, displayTable: displayTable, timeline: timelinePath, + output: output & "/StackSourceIP-Addresses.csv") # stack-ip-addresses -t ../hayabusa/timeline.jsonl --level --targetIpAddresses -o case-1/TargetIP-Addresses.csv - let cmd11 = StackIpAddressesCmd(name: "stack-ip-addresses(target)", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackTargetIP-Addresses.csv", targetIpAddresses:true) + let cmd11 = StackIpAddressesCmd(name: "stack-ip-addresses(target)", + level: level, displayTable: displayTable, timeline: timelinePath, + output: output & "/StackTargetIP-Addresses.csv", + targetIpAddresses: true) # stack-logons -t ../hayabusa/timeline.jsonl -o case-1/Logons.csv - let cmd12 = StackLogonsCmd(name: "stack-logons", displayTable:displayTable, timeline:timeline, output:output & "/StackLogons.csv") + let cmd12 = StackLogonsCmd(name: "stack-logons", displayTable: displayTable, + timeline: timelinePath, output: output & "/StackLogons.csv") # stack-processes -t ../hayabusa/timeline.jsonl --level -o case-1/Processes.csv - let cmd13 = StackProcessesCmd(name: "stack-processes", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackProcesses.csv") + let cmd13 = StackProcessesCmd(name: "stack-processes", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & "/StackProcesses.csv") # stack-services -t ../hayabusa/timeline.jsonl --level -o case-1/Services.csv - let cmd14 = StackServicesCmd(name:"stack-services", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackServices.csv") + let cmd14 = StackServicesCmd(name: "stack-services", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & "/StackServices.csv") # stack-tasks -t ../hayabusa/timeline.jsonl --level -o case-1/Tasks.csv - let cmd15 = StackTasksCmd(name:"stack-tasks", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackTasks.csv") + let cmd15 = StackTasksCmd(name: "stack-tasks", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & "/StackTasks.csv") # stack-users -t ../hayabusa/timeline.jsonl --level -o case-1/TargetUsers.csv - let cmd16 = StackUsersCmd(name:"stack-users(target)", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackTargetUsers.csv" , filterSystemAccounts:true, filterComputerAccounts:true) + let cmd16 = StackUsersCmd(name: "stack-users(target)", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & + "/StackTargetUsers.csv", filterSystemAccounts: true, + filterComputerAccounts: true) # stack-users -t ../hayabusa/timeline.jsonl --level --filterSystemAccounts=false --filterComputerAccounts=false -o case-1/TargetUsers-NoFiltering.csv - let cmd17 = StackUsersCmd(name:"stack-users(no filtering)", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackTargetUsers-NoFiltering.csv", filterSystemAccounts:false, filterComputerAccounts:false) + let cmd17 = StackUsersCmd(name: "stack-users(no filtering)", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & + "/StackTargetUsers-NoFiltering.csv", filterSystemAccounts: false, + filterComputerAccounts: false) # stack-users -t ../hayabusa/timeline.jsonl --level --sourceUsers -o users.csv - let cmd18 = StackUsersCmd(name:"stack-users(source)", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackSourceUsers.csv", sourceUsers:true, filterSystemAccounts:true, filterComputerAccounts:true) + let cmd18 = StackUsersCmd(name: "stack-users(source)", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & + "/StackSourceUsers.csv", sourceUsers: true, filterSystemAccounts: true, + filterComputerAccounts: true) # stack-users -t ../hayabusa/timeline.jsonl --level --sourceUsers --filterSystemAccounts=false --filterComputerAccounts=false -o case-1/SourceUsers-NoFiltering.csv - let cmd19 = StackUsersCmd(name:"stack-users(no filtering)", level:level, displayTable:displayTable, timeline:timeline, output:output & "/StackSourceUsers-NoFiltering.csv", sourceUsers:true, filterSystemAccounts:false, filterComputerAccounts:false) + let cmd19 = StackUsersCmd(name: "stack-users(no filtering)", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & + "/StackSourceUsers-NoFiltering.csv", sourceUsers: true, + filterSystemAccounts: false, filterComputerAccounts: false) - # timeline-logon -t ../hayabusa/timeline.jsonl -o case-1/LogonTimeline.csv - let cmd20 = TimelineLogonCmd(name:"timeline-logon", displayTable:displayTable, timeline:timeline, output:output & "/TimelineLogon.csv") + # timelinePath-logon -t ../hayabusa/timeline.jsonl -o case-1/LogonTimeline.csv + let cmd20 = TimelineLogonCmd(name: "timelinePath-logon", + displayTable: displayTable, timeline: timelinePath, output: output & "/TimelineLogon.csv") - # timeline-partition-diagnostic -t ../hayabusa/timeline.jsonl -o case-1/PartitionDiagnosticTimeline.csv - let cmd21 = TimelinePartitionDiagnosticCmd(name:"timeline-partition-diagnostic",displayTable:displayTable, timeline:timeline, output:output & "/TimelinePartitionDiagnostic.csv") + # timelinePath-partition-diagnostic -t ../hayabusa/timeline.jsonl -o case-1/PartitionDiagnosticTimeline.csv + let cmd21 = TimelinePartitionDiagnosticCmd( + name: "timelinePath-partition-diagnostic", displayTable: displayTable, + timeline: timelinePath, output: output & "/TimelinePartitionDiagnostic.csv") - # timeline-suspicious-processes -t ../hayabusa/timeline.jsonl --level -o case-1/SuspiciousProcesses.csv - let cmd22 = TimelineSuspiciousProcessesCmd(name:"timeline-suspicious-processes", level:level, displayTable:displayTable, timeline:timeline, output:output & "/TimelineSuspiciousProcesses.csv") + # timelinePath-suspicious-processes -t ../hayabusa/timeline.jsonl --level -o case-1/SuspiciousProcesses.csv + let cmd22 = TimelineSuspiciousProcessesCmd( + name: "timelinePath-suspicious-processes", level: level, + displayTable: displayTable, timeline: timelinePath, output: output & "/TimelineSuspiciousProcesses.csv") - # timeline-tasks -t ../hayabusa/timeline.jsonl -o case-1/TaskTimeline.csv - let cmd23 = TimelineTasksCmd(name:"timeline-tasks", displayTable:displayTable, timeline:timeline, output:output & "/TimelineTask.csv") + # timelinePath-tasks -t ../hayabusa/timeline.jsonl -o case-1/TaskTimeline.csv + let cmd23 = TimelineTasksCmd(name: "timelinePath-tasks", + displayTable: displayTable, timeline: timelinePath, output: output & "/TimelineTask.csv") # ttp-summary -t ../hayabusa/timeline.jsonl -o case-1/TTP-Summary.csv - let cmd24 = TTPSummaryCmd(name:"ttp-summary", displayTable:displayTable, timeline:timeline, output:output & "/TTPSummary.csv") + let cmd24 = TTPSummaryCmd(name: "ttp-summary", displayTable: displayTable, + timeline: timelinePath, output: output & "/TTPSummary.csv") cmd24.attack = readJsonFromFile("mitre-attack.json") # ttp-visualize -t ../hayabusa/timeline.jsonl -o case-1/MitreTTP-Heatmap.json - let cmd25 = TTPVisualizeCmd(name:"ttp-visualize", displayTable:displayTable, timeline:timeline, output:output & "/MitreTTP-Heatmap.json") + let cmd25 = TTPVisualizeCmd(name: "ttp-visualize", + displayTable: displayTable, timeline: timelinePath, output: output & "/MitreTTP-Heatmap.json") # execute all command let cmds = @[cmd1, cmd2, cmd3, cmd4, cmd5, cmd6, cmd7, cmd8, cmd9, cmd10, - cmd11, cmd12, cmd13, cmd14, cmd15, cmd16, cmd17, cmd18, cmd19, cmd20, + cmd11, cmd12, cmd13, cmd14, cmd15, cmd16, cmd17, cmd18, cmd19, + cmd20, cmd21, cmd22, cmd23, cmd24, cmd25] - let cmd = AutoMagicCmd(level: level, skipProgressBar:skipProgressBar, displayTable:displayTable, output: output, timeline: timeline, name:"automagic", msg:AutoMagicMsg) - cmd.analyzeJSONLFile(cmds) \ No newline at end of file + let cmd = AutoMagicCmd(level: level, skipProgressBar: skipProgressBar, + displayTable: displayTable, output: output, timeline: timelinePath, + name: "automagic", msg: AutoMagicMsg) + cmd.analyzeJSONLFile(cmds) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index b6dfcd68..ae1a83a6 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -12,15 +12,17 @@ proc outputScriptText(output: string, timestamp: string, computerName: string, scriptObj: Script) = var scriptText = "" for text in scriptObj.scriptBlocks.items: - scriptText = scriptText & text.replace("\\r\\n", "\p").replace("\\n", "\p").replace("\\t", "\t").replace("\\\"", "\"") + scriptText = scriptText & text.replace("\\r\\n", "\p").replace("\\n", + "\p").replace("\\t", "\t").replace("\\\"", "\"") let date = timestamp.replace(":", "_").replace(" ", "_") - let fileName = output & "/" & computerName & "-" & date & "-" & scriptObj.scriptBlockId & ".txt" - var outputFile = open(filename, fmWrite) + let fileName = output & "/" & computerName & "-" & date & "-" & + scriptObj.scriptBlockId & ".txt" + var outputFile = open(filename, fmAppend) outputFile.write(scriptText) flushFile(outputFile) close(outputFile) -proc calcMaxAlert(levels:HashSet):string = +proc calcMaxAlert(levels: HashSet): string = if "crit" in levels: return "crit" if "high" in levels: @@ -45,14 +47,14 @@ proc buildSummaryRecord(path: string, messageTotal: int, return [ts, cs, id, path, records, maxLevel, ruleTitles] type - ExtractScriptBlocksCmd* = ref object of AbstractCmd - currentIndex* = 0 - totalLines* = 0 - stackedRecords* = newTable[string, Script]() - summaryRecords* = newOrderedTable[string, array[7, string]]() - level*: string + ExtractScriptBlocksCmd* = ref object of AbstractCmd + currentIndex* = 0 + totalLines* = 0 + stackedRecords* = newTable[string, Script]() + summaryRecords* = newOrderedTable[string, array[7, string]]() + level*: string -method filter*(self: ExtractScriptBlocksCmd, x: HayabusaJson):bool = +method filter*(self: ExtractScriptBlocksCmd, x: HayabusaJson): bool = inc self.currentIndex return x.EventID == 4104 and isMinLevel(x.Level, self.level) @@ -76,22 +78,25 @@ method analyze*(self: ExtractScriptBlocksCmd, x: HayabusaJson) = self.stackedRecords[scriptBlockId].ruleTitles.incl(ruleTitle) self.stackedRecords[scriptBlockId].scriptBlocks.incl(scriptBlock) else: - self.stackedRecords[scriptBlockId] = Script(firstTimestamp: timestamp, - computerName: computerName, + self.stackedRecords[scriptBlockId] = Script( + firstTimestamp: timestamp, computerName: computerName, scriptBlockId: scriptBlockId, scriptBlocks: toOrderedSet([scriptBlock]), - levels:toHashSet([eventLevel]), ruleTitles:toHashSet([ruleTitle])) + levels: toHashSet([eventLevel]), ruleTitles: toHashSet([ruleTitle])) let scriptObj = self.stackedRecords[scriptBlockId] if messageNumber == messageTotal: if scriptBlockId in self.summaryRecords: - self.summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) + self.summaryRecords[scriptBlockId] = buildSummaryRecord(path, + messageTotal, scriptObj) # Already outputted discard outputScriptText(self.output, timestamp, computerName, scriptObj) - self.summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) + self.summaryRecords[scriptBlockId] = buildSummaryRecord(path, + messageTotal, scriptObj) elif self.currentIndex + 1 == self.totalLines: outputScriptText(self.output, timestamp, computerName, scriptObj) - self.summaryRecords[scriptBlockId] = buildSummaryRecord(path, messageTotal, scriptObj) + self.summaryRecords[scriptBlockId] = buildSummaryRecord(path, + messageTotal, scriptObj) except CatchableError: discard @@ -103,8 +108,9 @@ method resultOutput*(self: ExtractScriptBlocksCmd) = echo "No malicious powershell script block were found. There are either no malicious powershell script block or you need to change the level." else: let summaryFile = self.output & "/" & "Summary.csv" - let header = ["Creation Time", "Computer Name", "Script ID", "Script Name", "Records", "Level", "Alerts"] - var outputFile = open(summaryFile, fmWrite) + let header = ["Creation Time", "Computer Name", "Script ID", + "Script Name", "Records", "Level", "Alerts"] + var outputFile = open(summaryFile, fmAppend) var table: TerminalTable table.add header for i, val in header: @@ -114,7 +120,8 @@ method resultOutput*(self: ExtractScriptBlocksCmd) = outputFile.write(escapeCsvField(val) & "\p") for v in self.summaryRecords.values: let color = levelColor(v[5]) - table.add color v[0], color v[1], color v[2], color v[3], color v[4], color v[5], color v[6] + table.add color v[0], color v[1], color v[2], color v[3], color v[ + 4], color v[5], color v[6] for i, cell in v: if i < 6: outputFile.write(escapeCsvField(cell) & ",") @@ -122,28 +129,35 @@ method resultOutput*(self: ExtractScriptBlocksCmd) = outputFile.write(escapeCsvField(cell) & "\p") let outputFileSize = getFileSize(outputFile) outputFile.close() - savedFiles = padString(summaryFile & " (" & formatFileSize(outputFileSize) & ")", ' ', 80) - results = "PowerShell logs: " & intToStr(self.summaryRecords.len).insertSep(',') + savedFiles = padString(summaryFile & " (" & formatFileSize( + outputFileSize) & ")", ' ', 80) + results = "PowerShell logs: " & intToStr( + self.summaryRecords.len).insertSep(',') if self.displayTable: table.echoTableSepsWithStyled(seps = boxSeps) echo "" echo "The extracted PowerShell ScriptBlock is saved in the directory: " & self.output echo "Saved summary file: " & savedFiles - self.cmdResult = CmdResult(results:results, savedFiles: savedFiles & self.output & "/*.txt") + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles & + self.output & "/*.txt") -proc extractScriptblocks(level: string = "low", skipProgressBar:bool = false, output: string = "scriptblock-logs", quiet: bool = false, timeline: string) = +proc extractScriptblocks(level: string = "low", skipProgressBar: bool = false, + output: string = "scriptblock-logs", quiet: bool = false, + timeline: string) = checkArgs(quiet, timeline, level) - if not dirExists(output): - echo "The directory '" & output & "' does not exist so will be created." - createDir(output) - echo "" - let cmd = ExtractScriptBlocksCmd( - level: level, - skipProgressBar: skipProgressBar, - timeline: timeline, - output: output, - name:"extract-scriptblocks", - msg: ExtractScriptBlocksMsg) - cmd.totalLines = countLinesInTimeline(timeline, true) - cmd.analyzeJSONLFile() \ No newline at end of file + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: + if not dirExists(output): + echo "The directory '" & output & "' does not exist so will be created." + createDir(output) + echo "" + let cmd = ExtractScriptBlocksCmd( + level: level, + skipProgressBar: skipProgressBar, + timeline: timelinePath, + output: output, + name: "extract-scriptblocks", + msg: ExtractScriptBlocksMsg) + cmd.totalLines = countLinesInTimeline(timeline, true) + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/general.nim b/src/takajopkg/general.nim index 03fd850f..bf4048af 100644 --- a/src/takajopkg/general.nim +++ b/src/takajopkg/general.nim @@ -13,7 +13,8 @@ import takajoTerminal from std/streams import newFileStream -proc getJsonValue*(jsonResponse: JsonNode, keys: seq[string], default: string = "Unknown"): string = +proc getJsonValue*(jsonResponse: JsonNode, keys: seq[string], + default: string = "Unknown"): string = var value = jsonResponse for key in keys: if value.kind == JObject and value.hasKey(key): @@ -21,11 +22,11 @@ proc getJsonValue*(jsonResponse: JsonNode, keys: seq[string], default: string = else: return default if value.kind == JInt: - return $value.getInt() # Convert to string + return $value.getInt() # Convert to string elif value.kind == JString: return value.getStr() else: - return default # If it's neither a string nor an integer, return default + return default # If it's neither a string nor an integer, return default proc getJsonDate*(jsonResponse: JsonNode, keys: seq[string]): string = @@ -52,12 +53,22 @@ proc getFileNameWithExt*(targetPath: string): string = file &= ext return file -proc getTargetExtFileLists*(targetDirPath: string, targetExt: string): seq[string] = +proc getTargetExtFileLists*(targetDirPath: string, targetExt: string, + fullPathOutput: bool): seq[string] = ## extract yml file name seq to specified directory path var r: seq[string] = @[] + if fileExists(targetDirPath): + if targetDirPath.endsWith(targetExt): + if fullPathOutput: + r.insert(targetDirPath) + else: + r.insert(getFileNameWithExt(targetDirPath)) for f in walkDirRec(targetDirPath): if f.endsWith(targetExt): - r.insert(getFileNameWithExt(f)) + if fullPathOutput: + r.insert(f) + else: + r.insert(getFileNameWithExt(f)) # removed duplicated file name from seq r = deduplicate(r) if r.len() == 0: @@ -65,7 +76,8 @@ proc getTargetExtFileLists*(targetDirPath: string, targetExt: string): seq[strin else: r -proc getHayabusaCsvData*(csvPath: string, columnName: string): Tableref[string, seq[string]] = +proc getHayabusaCsvData*(csvPath: string, columnName: string): Tableref[string, + seq[string]] = ## procedure for Hayabusa output csv read data. var s = newFileStream(csvPath, fmRead) @@ -101,7 +113,8 @@ proc formatDuration*(d: Duration): string = minutes = d.inMinutes mod 60 seconds = d.inSeconds mod 60 milliseconds = d.inMilliseconds mod 1000 - return $days & "d " & $hours & "h " & $minutes & "m " & $seconds & "s " & $milliseconds & "ms" + return $days & "d " & $hours & "h " & $minutes & "m " & $seconds & "s " & + $milliseconds & "ms" proc extractStr*(jsonObj: JsonNode, key: string): string = let value = jsonObj.hasKey(key) @@ -202,7 +215,7 @@ proc elevatedTokenIdToName*(elevatedTokenId: string): string = return result proc getYAMLpathes*(rulesDir: string): seq[string] = - var pathes:seq[string] = @[] + var pathes: seq[string] = @[] for entry in walkDirRec(rulesDir): if entry.endsWith(".yml"): pathes.add(entry) @@ -229,11 +242,11 @@ proc formatFileSize*(fileSize: BiggestInt): string = fileSizeStr = $fileSize & " Bytes" return fileSizeStr -proc countLinesInTimeline*(filePath: string, quiet:bool = false): int = +proc countLinesInTimeline*(filePath: string, quiet: bool = false): int = echo "File: " & filePath & " (" & formatFileSize(getFileSize(filePath)) & ")" if not quiet: - echo "Counting total lines. Please wait." - const BufferSize = 4 * 1024 * 1024 # 4 MiB + echo "Counting total lines. Please wait." + const BufferSize = 4 * 1024 * 1024 # 4 MiB var buffer = newString(BufferSize) var file = open(filePath) var count = 0 @@ -261,22 +274,24 @@ proc isMinLevel*(levelInLog: string, userSetLevel: string): bool = of "medium": return levelInLog == "crit" or levelInLog == "high" or levelInLog == "med" of "low": - return levelInLog == "crit" or levelInLog == "high" or levelInLog == "med" or levelInLog == "low" + return levelInLog == "crit" or levelInLog == "high" or levelInLog == + "med" or levelInLog == "low" of "informational": - return levelInLog == "crit" or levelInLog == "high" or levelInLog == "med" or levelInLog == "low" or levelInLog == "info" + return levelInLog == "crit" or levelInLog == "high" or levelInLog == + "med" or levelInLog == "low" or levelInLog == "info" else: return false proc isPrivateIP*(ip: string): bool = - let - ipv4Private = re"^(10\.\d{1,3}\.\d{1,3}\.\d{1,3})$|^(192\.168\.\d{1,3}\.\d{1,3})$|^(172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})$|^(127\.\d{1,3}\.\d{1,3}\.\d{1,3})$" - ipv4MappedIPv6Private = re"^::ffff:(10\.\d{1,3}\.\d{1,3}\.\d{1,3})$|^::ffff:(192\.168\.\d{1,3}\.\d{1,3})$|^::ffff:(172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})$|^::ffff:(127\.\d{1,3}\.\d{1,3}\.\d{1,3})$" - ipv6Private = re"^fd[0-9a-f]{2}:|^fe80:" - - if ip =~ ipv4Private or ip =~ ipv4MappedIPv6Private or ip =~ ipv6Private: - return true - else: - return false + let + ipv4Private = re"^(10\.\d{1,3}\.\d{1,3}\.\d{1,3})$|^(192\.168\.\d{1,3}\.\d{1,3})$|^(172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})$|^(127\.\d{1,3}\.\d{1,3}\.\d{1,3})$" + ipv4MappedIPv6Private = re"^::ffff:(10\.\d{1,3}\.\d{1,3}\.\d{1,3})$|^::ffff:(192\.168\.\d{1,3}\.\d{1,3})$|^::ffff:(172\.(1[6-9]|2\d|3[01])\.\d{1,3}\.\d{1,3})$|^::ffff:(127\.\d{1,3}\.\d{1,3}\.\d{1,3})$" + ipv6Private = re"^fd[0-9a-f]{2}:|^fe80:" + + if ip =~ ipv4Private or ip =~ ipv4MappedIPv6Private or ip =~ ipv6Private: + return true + else: + return false proc isMulticast*(address: string): bool = # Define regex for IPv4 multicast addresses @@ -285,7 +300,8 @@ proc isMulticast*(address: string): bool = # Define regex for IPv6 multicast addresses let ipv6MulticastPattern = re"(?i)^ff[0-9a-f]{2}:" # check if the address matches either of the multicast patterns - if address.find(ipv4MulticastPattern) >= 0 or address.find(ipv6MulticastPattern) >= 0: + if address.find(ipv4MulticastPattern) >= 0 or address.find( + ipv6MulticastPattern) >= 0: return true return false @@ -297,7 +313,8 @@ proc isLoopback*(address: string): bool = let ipv6LoopbackPattern = re"^(?:0*:)*?:?0*1$" # Check if the address matches either of the loopback patterns - if address.find(ipv4LoopbackPattern) >= 0 or address.find(ipv6LoopbackPattern) >= 0: + if address.find(ipv4LoopbackPattern) >= 0 or address.find( + ipv6LoopbackPattern) >= 0: return true return false @@ -306,8 +323,10 @@ proc isIpAddress*(s: string): bool = return s.find(ipRegex) != -1 proc extractDomain*(domain: string): string = - let doubleTLDs = ["co.jp", "co.uk", "com.au", "org.au", "net.au", "com.br", "net.br", "com.cn", "net.cn", - "com.mx", "net.mx", "ac.nz", "co.nz", "net.nz", "co.za", "net.za", "co.in", "net.in", "ac.uk", "gov.uk"] + let doubleTLDs = ["co.jp", "co.uk", "com.au", "org.au", "net.au", "com.br", + "net.br", "com.cn", "net.cn", + "com.mx", "net.mx", "ac.nz", "co.nz", "net.nz", "co.za", "net.za", + "co.in", "net.in", "ac.uk", "gov.uk"] let parts = domain.split('.') if parts.len >= 3: let lastTwo = parts[^2] & '.' & parts[^1] @@ -318,27 +337,27 @@ proc extractDomain*(domain: string): string = return domain proc isLocalIP*(ip: string): bool = - return ip == "127.0.0.1" or ip == "-" or ip == "::1" - -proc isJsonConvertible*(timeline: string) : bool = - var - file: File - firstLine: string - jsonLine: JsonNode - if file.open(timeline): - try: - firstLine = file.readLine() - jsonLine = parseJson(firstLine) - return true - except CatchableError: - echo "Failed to open '" & timeline & "' because it is not in JSONL format." - echo "Please specify a JSONL-formatted file that has been created with the Hayabusa json-timeline command and -L or --JSONL-output option." - echo "" - return false - finally: - close(file) - echo "Failed to open '" & timeline & "'. Please specify a valid file path.\p" - return false + return ip == "127.0.0.1" or ip == "-" or ip == "::1" + +proc isJsonConvertible*(timeline: string): bool = + var + file: File + firstLine: string + jsonLine: JsonNode + if file.open(timeline): + try: + firstLine = file.readLine() + jsonLine = parseJson(firstLine) + return true + except CatchableError: + echo "Failed to open '" & timeline & "' because it is not in JSONL format." + echo "Please specify a JSONL-formatted file that has been created with the Hayabusa json-timeline command and -L or --JSONL-output option." + echo "" + return false + finally: + close(file) + echo "Failed to open '" & timeline & "'. Please specify a valid file path.\p" + return false proc outputElapsedTime*(startTime: float) = let endTime = epochTime() @@ -347,27 +366,30 @@ proc outputElapsedTime*(startTime: float) = let minutes = (elapsedTime2 mod 3600) div 60 let seconds = elapsedTime2 mod 60 echo "" - echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & $seconds & " seconds" + echo "Elapsed time: ", $hours & " hours, " & $minutes & " minutes, " & + $seconds & " seconds" echo "" -proc checkArgs*(quiet: bool = false, timeline: string, level:string) = +proc checkArgs*(quiet: bool = false, timeline: string, level: string) = if not quiet: styledEcho(fgGreen, outputLogo()) - if not os.fileExists(timeline): + if not os.fileExists(timeline) or os.dirExists(timeline): echo "The file '" & timeline & "' does not exist. Please specify a valid file path." quit(1) if not isJsonConvertible(timeline): quit(1) - if level != "critical" and level != "high" and level != "medium" and level != "low" and level != "informational": + if level != "critical" and level != "high" and level != "medium" and + level != "low" and level != "informational": echo "You must specify a minimum level of critical, high, medium, low or informational. (default: low)" echo "" quit(1) -proc countJsonlAndStartMsg*(cmdName:string, msg:string, timeline:string):int = +proc countJsonlAndStartMsg*(cmdName: string, msg: string, + timeline: string): int = echo "Started the " & cmdName & " command" echo "" echo msg @@ -393,7 +415,7 @@ proc padString*(s: string, padChar: char, format: string): string = let padding = repeat($padChar, length - len(s)) return strip(s & padding) -proc getTimeFormat*(ts:seq[TableRef[string, string]]): string = +proc getTimeFormat*(ts: seq[TableRef[string, string]]): string = if ts.len() == 0: return "" let t = ts[0]["Timestamp"] @@ -401,13 +423,13 @@ proc getTimeFormat*(ts:seq[TableRef[string, string]]): string = if t.endsWith("Z"): # --ISO-8601 return "yyyy-MM-dd'T'HH:mm:ss'.'ffffff" elif t.contains("AM") or t.contains("PM"): # --US-time - return "MM-dd-yyyy HH:mm:ss'.'fff tt" + return "MM-dd-yyyy HH:mm:ss'.'fff tt" elif t.contains(","): # --RFC-2822 return "ddd,d MMM yyyy HH:mm:ss" elif t.count(' ') == 1: # --RFC-3339 return "yyyy-MM-dd HH:mm:ss'.'ffffff" - elif t[4] == '-' : # -UTC - return "yyyy-MM-dd HH:mm:ss'.'fff" + elif t[4] == '-': # -UTC + return "yyyy-MM-dd HH:mm:ss'.'fff" # --European-time/--US-military-timeは月以外のフォーマットは同じなので、パースエラーの有無でタイムフォーマットを判定 for s in ts: try: @@ -415,4 +437,4 @@ proc getTimeFormat*(ts:seq[TableRef[string, string]]): string = let x = parse(padString(s["Timestamp"], '0', timeFormat), timeFormat) except CatchableError: return "dd-MM-yyyy HH:mm:ss'.'fff" # --European-time - return "MM-dd-yyyy HH:mm:ss'.'fff" # --US-military-time \ No newline at end of file + return "MM-dd-yyyy HH:mm:ss'.'fff" # --US-military-time diff --git a/src/takajopkg/listDomains.nim b/src/takajopkg/listDomains.nim index de5960ec..35bdf674 100644 --- a/src/takajopkg/listDomains.nim +++ b/src/takajopkg/listDomains.nim @@ -8,46 +8,52 @@ Domains ending with .lan, .LAN or .local are filtered out.""" type ListDomainsCmd* = ref object of AbstractCmd - domainHashSet* = initHashSet[string]() - includeSubdomains*: bool - includeWorkstations*: bool + domainHashSet* = initHashSet[string]() + includeSubdomains*: bool + includeWorkstations*: bool -method filter*(self: ListDomainsCmd, x: HayabusaJson):bool = - if not (x.Channel == "Sysmon" and x.EventId == 22): - return false - let domain = x.Details.extractStr("Query") - return self.includeWorkstations or (domain.contains('.') and domain != "." and not domain.endsWith(".lan") and - not domain.endsWith(".LAN") and not domain.endsWith(".local") and not isIpAddress(domain) and not domain.endsWith('.')) +method filter*(self: ListDomainsCmd, x: HayabusaJson): bool = + if not (x.Channel == "Sysmon" and x.EventId == 22): + return false + let domain = x.Details.extractStr("Query") + return self.includeWorkstations or (domain.contains('.') and domain != "." and + not domain.endsWith(".lan") and + not domain.endsWith(".LAN") and not domain.endsWith(".local") and + not isIpAddress(domain) and not domain.endsWith('.')) method analyze*(self: ListDomainsCmd, x: HayabusaJson) = - var domain = x.Details.extractStr("Query") - if not self.includeSubdomains: - domain = extractDomain(domain) - self.domainHashSet.incl(domain) + var domain = x.Details.extractStr("Query") + if not self.includeSubdomains: + domain = extractDomain(domain) + self.domainHashSet.incl(domain) -method resultOutput*(self: ListDomainsCmd)= - # Save results - var outputFile = open(self.output, fmWrite) - for domain in self.domainHashSet: - outputFile.write(domain & "\p") - let outputFileSize = getFileSize(outputFile) - outputFile.close() - let savedFiles = self.output & " (" & formatFileSize(outputFileSize) & ")" - let results = "Domains: " & intToStr(len(self.domainHashSet)).insertSep(',') - if self.displayTable: - echo "" - echo results - echo "Saved file: " & savedFiles - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) +method resultOutput*(self: ListDomainsCmd) = + # Save results + var outputFile = open(self.output, fmAppend) + for domain in self.domainHashSet: + outputFile.write(domain & "\p") + let outputFileSize = getFileSize(outputFile) + outputFile.close() + let savedFiles = self.output & " (" & formatFileSize(outputFileSize) & ")" + let results = "Domains: " & intToStr(len(self.domainHashSet)).insertSep(',') + if self.displayTable: + echo "" + echo results + echo "Saved file: " & savedFiles + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc listDomains(includeSubdomains: bool = false, includeWorkstations: bool = false, skipProgressBar:bool = false, output: string, quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, "informational") +proc listDomains(includeSubdomains: bool = false, + includeWorkstations: bool = false, skipProgressBar: bool = false, + output: string, quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, "informational") + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = ListDomainsCmd( skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, - name:"list-domains", + name: "list-domains", msg: ListDomainsMsg, includeSubdomains: includeSubdomains, includeWorkstations: includeWorkstations) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/listHashes.nim b/src/takajopkg/listHashes.nim index 185c46a9..955033f6 100644 --- a/src/takajopkg/listHashes.nim +++ b/src/takajopkg/listHashes.nim @@ -9,100 +9,109 @@ For example, -l=informational for a minimum level of informational, which will e type ListHashesCmd* = ref object of AbstractCmd - level:string - md5hashes*, sha1hashes*, sha256hashes*, impHashes* = initHashSet[string]() - md5hashCount*, sha1hashCount*, sha256hashCount*, impHashCount* = 0 + level: string + md5hashes*, sha1hashes*, sha256hashes*, impHashes * = initHashSet[string]() + md5hashCount*, sha1hashCount*, sha256hashCount*, impHashCount * = 0 -method filter*(self: ListHashesCmd, x: HayabusaJson):bool = - return x.Channel == "Sysmon" and x.EventID == 1 and isMinLevel(x.Level, self.level) +method filter*(self: ListHashesCmd, x: HayabusaJson): bool = + return x.Channel == "Sysmon" and x.EventID == 1 and isMinLevel(x.Level, self.level) method analyze*(self: ListHashesCmd, x: HayabusaJson) = - try: - let hashes = x.Details["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist. - let pairs = hashes.split(",") # Split the string into key-value pairs. Ex: MD5=DE9C75F34F47B60A71BBA03760F0579E,SHA256=12F06D3B1601004DB3F7F1A07E7D3AF4CC838E890E0FF50C51E4A0C9366719ED,IMPHASH=336674CB3C8337BDE2C22255345BFF43 - for pair in pairs: - let keyVal = pair.split("=") - case keyVal[0]: - of "MD5": - self.md5hashes.incl(keyVal[1]) - inc self.md5hashCount - of "SHA1": - self.sha1hashes.incl(keyVal[1]) - inc self.sha1hashCount - of "SHA256": - self.sha256hashes.incl(keyVal[1]) - inc self.sha256hashCount - of "IMPHASH": - self.impHashes.incl(keyVal[1]) - inc self.impHashCount - except KeyError: - discard + try: + let hashes = x.Details["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist. + let pairs = hashes.split(",") # Split the string into key-value pairs. Ex: MD5=DE9C75F34F47B60A71BBA03760F0579E,SHA256=12F06D3B1601004DB3F7F1A07E7D3AF4CC838E890E0FF50C51E4A0C9366719ED,IMPHASH=336674CB3C8337BDE2C22255345BFF43 + for pair in pairs: + let keyVal = pair.split("=") + case keyVal[0]: + of "MD5": + self.md5hashes.incl(keyVal[1]) + inc self.md5hashCount + of "SHA1": + self.sha1hashes.incl(keyVal[1]) + inc self.sha1hashCount + of "SHA256": + self.sha256hashes.incl(keyVal[1]) + inc self.sha256hashCount + of "IMPHASH": + self.impHashes.incl(keyVal[1]) + inc self.impHashCount + except KeyError: + discard method resultOutput*(self: ListHashesCmd) = - let output = self.output - let md5outputFilename = output & "-MD5.txt" - var md5outputFile = open(md5outputFilename, fmWrite) - for hash in self.md5hashes: - md5outputFile.write(hash & "\p") - md5outputFile.close() - let md5FileSize = getFileSize(md5outputFilename) + let output = self.output + let md5outputFilename = output & "-MD5.txt" + var md5outputFile = open(md5outputFilename, fmWrite) + for hash in self.md5hashes: + md5outputFile.write(hash & "\p") + md5outputFile.close() + let md5FileSize = getFileSize(md5outputFilename) - # Save SHA1 results - let sha1outputFilename = self.output & "-SHA1.txt" - var sha1outputFile = open(sha1outputFilename, fmWrite) - for hash in self.sha1hashes: - sha1outputFile.write(hash & "\p") - sha1outputFile.close() - let sha1FileSize = getFileSize(sha1outputFilename) + # Save SHA1 results + let sha1outputFilename = self.output & "-SHA1.txt" + var sha1outputFile = open(sha1outputFilename, fmWrite) + for hash in self.sha1hashes: + sha1outputFile.write(hash & "\p") + sha1outputFile.close() + let sha1FileSize = getFileSize(sha1outputFilename) - # Save SHA256 results - let sha256outputFilename = output & "-SHA256.txt" - var sha256outputFile = open(sha256outputFilename, fmWrite) - for hash in self.sha256hashes: - sha256outputFile.write(hash & "\p") - sha256outputFile.close() - let sha256FileSize = getFileSize(sha256outputFilename) + # Save SHA256 results + let sha256outputFilename = output & "-SHA256.txt" + var sha256outputFile = open(sha256outputFilename, fmWrite) + for hash in self.sha256hashes: + sha256outputFile.write(hash & "\p") + sha256outputFile.close() + let sha256FileSize = getFileSize(sha256outputFilename) - # Save IMPHASH results - let impHashOutputFilename = output & "-ImportHashes.txt" - var impHashOutputFile = open(impHashOutputFilename, fmWrite) - for hash in self.impHashes: - impHashOutputFile.write(hash & "\p") - impHashOutputFile.close() - let impHashFileSize = getFileSize(impHashOutputFilename) - let savedFiles = "" & - padString(md5outputFilename & " (" & formatFileSize(md5FileSize) & ")",' ', 80) & - padString(sha1outputFilename & " (" & formatFileSize(sha1FileSize) & ")",' ', 80) & - padString(sha256outputFilename & " (" & formatFileSize(sha256FileSize) & ")",' ', 80) & - padString(impHashOutputFilename & " (" & formatFileSize(impHashFileSize) & ")", ' ', 80) - let results = "" & - padString("MD5: " & intToStr(self.md5hashCount).insertSep(','),' ', 80) & - padString("SHA1: " & intToStr(self.sha1hashCount).insertSep(','),' ', 80) & - padString("SHA256: " & intToStr(self.sha256hashCount).insertSep(','), ' ', 80) & - padString("Import: " & intToStr(self.impHashCount).insertSep(','), ' ', 80) - if self.displayTable: - echo "" - echo "Saved files:" - echo md5outputFilename & " (" & formatFileSize(md5FileSize) & ")" - echo sha1outputFilename & " (" & formatFileSize(sha1FileSize) & ")" - echo sha256outputFilename & " (" & formatFileSize(sha256FileSize) & ")" - echo impHashOutputFilename & " (" & formatFileSize(impHashFileSize) & ")" - echo "" - echo "Hashes:" - echo "MD5: ", intToStr(self.md5hashCount).insertSep(',') - echo "SHA1: ", intToStr(self.sha1hashCount).insertSep(',') - echo "SHA256: ", intToStr(self.sha256hashCount).insertSep(',') - echo "Import: ", intToStr(self.impHashCount).insertSep(',') - echo "" - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) + # Save IMPHASH results + let impHashOutputFilename = output & "-ImportHashes.txt" + var impHashOutputFile = open(impHashOutputFilename, fmWrite) + for hash in self.impHashes: + impHashOutputFile.write(hash & "\p") + impHashOutputFile.close() + let impHashFileSize = getFileSize(impHashOutputFilename) + let savedFiles = "" & + padString(md5outputFilename & " (" & formatFileSize(md5FileSize) & ")", + ' ', 80) & + padString(sha1outputFilename & " (" & formatFileSize(sha1FileSize) & ")", + ' ', 80) & + padString(sha256outputFilename & " (" & formatFileSize(sha256FileSize) & + ")", ' ', 80) & + padString(impHashOutputFilename & " (" & formatFileSize(impHashFileSize) & + ")", ' ', 80) + let results = "" & + padString("MD5: " & intToStr(self.md5hashCount).insertSep(','), ' ', 80) & + padString("SHA1: " & intToStr(self.sha1hashCount).insertSep(','), ' ', + 80) & + padString("SHA256: " & intToStr(self.sha256hashCount).insertSep(','), ' ', + 80) & + padString("Import: " & intToStr(self.impHashCount).insertSep(','), ' ', 80) + if self.displayTable: + echo "" + echo "Saved files:" + echo md5outputFilename & " (" & formatFileSize(md5FileSize) & ")" + echo sha1outputFilename & " (" & formatFileSize(sha1FileSize) & ")" + echo sha256outputFilename & " (" & formatFileSize(sha256FileSize) & ")" + echo impHashOutputFilename & " (" & formatFileSize(impHashFileSize) & ")" + echo "" + echo "Hashes:" + echo "MD5: ", intToStr(self.md5hashCount).insertSep(',') + echo "SHA1: ", intToStr(self.sha1hashCount).insertSep(',') + echo "SHA256: ", intToStr(self.sha256hashCount).insertSep(',') + echo "Import: ", intToStr(self.impHashCount).insertSep(',') + echo "" + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc listHashes(level: string = "high", skipProgressBar:bool = false, output: string, quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc listHashes(level: string = "high", skipProgressBar: bool = false, + output: string, quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = ListHashesCmd( skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, level: level, output: output, - name:"list-hashes", + name: "list-hashes", msg: ListHashesMsg) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/listIpAddresses.nim b/src/takajopkg/listIpAddresses.nim index 7371681e..27fd0943 100644 --- a/src/takajopkg/listIpAddresses.nim +++ b/src/takajopkg/listIpAddresses.nim @@ -5,27 +5,29 @@ Outbound traffic is included by default but can be disabled with -O=false. Private IP addresses are not included by default but can be enabled with -p.""" type - ListIpAddressesCmd* = ref object of AbstractCmd - ipHashSet* = initHashSet[string]() - inbound*: bool - outbound*: bool - privateIp*: bool + ListIpAddressesCmd* = ref object of AbstractCmd + ipHashSet* = initHashSet[string]() + inbound*: bool + outbound*: bool + privateIp*: bool -method filter*(self: ListIpAddressesCmd, x: HayabusaJson):bool = +method filter*(self: ListIpAddressesCmd, x: HayabusaJson): bool = return true method analyze*(self: ListIpAddressesCmd, x: HayabusaJson) = if self.inbound: let ipAddress = getJsonValue(x.Details, @["SrcIP"]) if (not isPrivateIP(ipAddress) or self.privateIp) and - not isMulticast(ipAddress) and not isLoopback(ipAddress) and ipAddress != "Unknown" and ipAddress != "-": + not isMulticast(ipAddress) and not isLoopback(ipAddress) and + ipAddress != "Unknown" and ipAddress != "-": self.ipHashSet.incl(ipAddress) # Search for events with a TgtIP field if outbound == true if self.outbound: let ipAddress = getJsonValue(x.Details, @["TgtIP"]) if (not isPrivateIP(ipAddress) or self.privateIp) and - not isMulticast(ipAddress) and not isLoopback(ipAddress) and ipAddress != "Unknown" and ipAddress != "-": + not isMulticast(ipAddress) and not isLoopback(ipAddress) and + ipAddress != "Unknown" and ipAddress != "-": self.ipHashSet.incl(ipAddress) method resultOutput*(self: ListIpAddressesCmd) = @@ -41,16 +43,20 @@ method resultOutput*(self: ListIpAddressesCmd) = echo "" echo "IP Addresss: ", intToStr(len(self.ipHashSet)).insertSep(',') echo "Saved file: " & savedFiles - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc listIpAddresses(inbound: bool = true, outbound: bool = true, skipProgressBar:bool = false, output: string, privateIp: bool = false, quiet: bool = false, timeline: string) = +proc listIpAddresses(inbound: bool = true, outbound: bool = true, + skipProgressBar: bool = false, output: string, privateIp: bool = false, + quiet: bool = false, timeline: string) = checkArgs(quiet, timeline, "informational") - let cmd = ListIpAddressesCmd( - timeline: timeline, - output: output, - name:"list-ip-addresses", - msg: ListIpAddressesMsg, - inbound: inbound, - outbound: outbound, - privateIp: privateIp) - cmd.analyzeJSONLFile() \ No newline at end of file + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: + let cmd = ListIpAddressesCmd( + timeline: timelinePath, + output: output, + name: "list-ip-addresses", + msg: ListIpAddressesMsg, + inbound: inbound, + outbound: outbound, + privateIp: privateIp) + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/listUndetectedEvtxFiles.nim b/src/takajopkg/listUndetectedEvtxFiles.nim index f2cfd949..b673bcc0 100644 --- a/src/takajopkg/listUndetectedEvtxFiles.nim +++ b/src/takajopkg/listUndetectedEvtxFiles.nim @@ -1,4 +1,6 @@ -proc listUndetectedEvtxFiles(columnName: system.string = "EvtxFile", evtxDir: string, output: string = "", quiet: bool = false, timeline: string) = +proc listUndetectedEvtxFiles(columnName: system.string = "EvtxFile", + evtxDir: string, output: string = "", quiet: bool = false, + timeline: string) = if not quiet: styledEcho(fgGreen, outputLogo()) @@ -8,7 +10,7 @@ proc listUndetectedEvtxFiles(columnName: system.string = "EvtxFile", evtxDir: st quit(1) let csvData: TableRef[string, seq[string]] = getHayabusaCsvData(timeline, columnName) - var fileLists: seq[string] = getTargetExtFileLists(evtxDir, ".evtx") + var fileLists: seq[string] = getTargetExtFileLists(evtxDir, ".evtx", false) var detectedPaths: seq[string] = csvData[columnName].map(getFileNameWithExt) detectedPaths = deduplicate(detectedPaths) @@ -44,4 +46,4 @@ proc listUndetectedEvtxFiles(columnName: system.string = "EvtxFile", evtxDir: st echo "" else: echo outputstock.join("\n") - discard \ No newline at end of file + discard diff --git a/src/takajopkg/listUnusedRules.nim b/src/takajopkg/listUnusedRules.nim index db3c6687..ef246704 100644 --- a/src/takajopkg/listUnusedRules.nim +++ b/src/takajopkg/listUnusedRules.nim @@ -1,4 +1,5 @@ -proc listUnusedRules(columnName: string = "RuleFile", output: string = "", quiet: bool = false, rulesDir: string, timeline: string) = +proc listUnusedRules(columnName: string = "RuleFile", output: string = "", + quiet: bool = false, rulesDir: string, timeline: string) = if not quiet: styledEcho(fgGreen, outputLogo()) @@ -7,7 +8,7 @@ proc listUnusedRules(columnName: string = "RuleFile", output: string = "", quiet quit(1) let csvData: TableRef[string, seq[string]] = getHayabusaCsvData(timeline, columnName) - var fileLists: seq[string] = getTargetExtFileLists(rulesDir, ".yml") + var fileLists: seq[string] = getTargetExtFileLists(rulesDir, ".yml", false) var detectedPaths: seq[string] = csvData[columnName].map(getFileNameWithExt) detectedPaths = deduplicate(detectedPaths) var outputStock: seq[string] = @[] @@ -41,4 +42,4 @@ proc listUnusedRules(columnName: string = "RuleFile", output: string = "", quiet echo "" else: echo outputstock.join("\n") - discard \ No newline at end of file + discard diff --git a/src/takajopkg/splitCsvTimeline.nim b/src/takajopkg/splitCsvTimeline.nim index 8eb52691..2c0717d8 100644 --- a/src/takajopkg/splitCsvTimeline.nim +++ b/src/takajopkg/splitCsvTimeline.nim @@ -1,11 +1,10 @@ -proc splitCsvTimeline(makeMultiline: bool = false, output: string = "output", quiet: bool = false, timeline: string) = +proc splitCsvTimeline(makeMultiline: bool = false, output: string = "output", + quiet: bool = false, timeline: string) = let startTime = epochTime() if not quiet: styledEcho(fgGreen, outputLogo()) - if not os.fileExists(timeline): - echo "The file '" & timeline & "' does not exist. Please specify a valid file path." - quit(1) + checkArgs(quiet, timeline, "informational") echo "Started the Split CSV Timeline command" echo "" @@ -13,7 +12,12 @@ proc splitCsvTimeline(makeMultiline: bool = false, output: string = "output", qu echo "If you want to separate the field data by newlines, add the -m option." echo "" - let totalLines = countLinesInTimeline(timeline) + var totalLines = 0 + var filePaths = getTargetExtFileLists(timeline, ".csv", true) + for timelinePath in filePaths: + totalLines += countLinesInTimeline(timelinePath) + + echo "Splitting the Hayabusa CSV timeline. Please wait." if not dirExists(output): @@ -23,67 +27,68 @@ proc splitCsvTimeline(makeMultiline: bool = false, output: string = "output", qu echo "" var - inputFile = open(timeline, FileMode.fmRead) - line = "" filenameSequence: seq[string] = @[] filesTable = initTable[string, File]() bar: SuruBar = initSuruBar() bar[0].total = totalLines - # Read in the CSV header - let csvHeader = inputFile.readLine() - let headerColums = csvHeader.split(',') - var computerColumnNum = 0 - for i, col in enumerate(headerColums): - if "Computer" in col: - computerColumnNum = i - break - if computerColumnNum == 0: + for timelinePath in filePaths: + var inputFile = open(timelinePath, fmRead) + # Read in the CSV header + let csvHeader = inputFile.readLine() + let headerColums = csvHeader.split(',') + var computerColumnNum = 0 + for i, col in enumerate(headerColums): + if "Computer" in col: + computerColumnNum = i + break + if computerColumnNum == 0: + echo "" + echo "There is no column for Computer. Please specify a valid csv." + createDir(output) echo "" - echo "There is no column for Computer. Please specify a valid csv." - createDir(output) - echo "" - bar.setup() + bar.setup() - inc bar - while inputFile.endOfFile == false: inc bar - bar.update(1000000000) # refresh every second - - var currentLine = inputFile.readLine() - let splitFields = currentLine.split(',') - var computerName = splitFields[computerColumnNum] - computerName = computerName[1 .. computerName.len - 2] # Remove surrounding double quotes - - # If it is the first time we see this computer name, then record it in a str sequence, create a file, - # write the CSV headers and current row. - if not filesTable.hasKey(computerName): - let filename = output & "/" & computerName & "-HayabusaResults.csv" - filenameSequence.add(filename) - var outputFile = open(filename, fmWrite) - filesTable[computerName] = outputFile - outputFile.write(csvHeader) + while inputFile.endOfFile == false: + inc bar + bar.update(1000000000) # refresh every second + + var currentLine = inputFile.readLine() + let splitFields = currentLine.split(',') + var computerName = splitFields[computerColumnNum] + computerName = computerName[1 .. computerName.len - + 2] # Remove surrounding double quotes + + # If it is the first time we see this computer name, then record it in a str sequence, create a file, + # write the CSV headers and current row. + if not filesTable.hasKey(computerName): + let filename = output & "/" & computerName & "-HayabusaResults.csv" + filenameSequence.add(filename) + var outputFile = open(filename, fmAppend) + filesTable[computerName] = outputFile + outputFile.write(csvHeader) + outputFile.write("\p") + flushFile(outputFile) # Flush buffer after writing to file + + # Use the file from the table and write the line. + var outputFile = filesTable[computerName] + if makeMultiline == true: + currentLine = currentLine.replace("¦", "\n") + outputFile.write(currentLine) + else: + outputFile.write(currentLine) outputFile.write("\p") - flushFile(outputFile) # Flush buffer after writing to file - - # Use the file from the table and write the line. - var outputFile = filesTable[computerName] - if makeMultiline == true: - currentLine = currentLine.replace("¦", "\n") - outputFile.write(currentLine) - else: - outputFile.write(currentLine) - outputFile.write("\p") - flushFile(outputFile) - inc bar - bar.finish() - - # Close all opened files - for file in filesTable.values: - close(file) - - close(inputFile) + flushFile(outputFile) + inc bar + bar.finish() + + # Close all opened files + for file in filesTable.values: + close(file) + + close(inputFile) echo "" for fn in filenameSequence: @@ -91,4 +96,4 @@ proc splitCsvTimeline(makeMultiline: bool = false, output: string = "output", qu echo "Saved file: " & fn & " (" & formatFileSize(fileSize) & ")" echo "" - outputElapsedTime(startTime) \ No newline at end of file + outputElapsedTime(startTime) diff --git a/src/takajopkg/splitJsonTimeline.nim b/src/takajopkg/splitJsonTimeline.nim index a95f20c8..b64d298e 100644 --- a/src/takajopkg/splitJsonTimeline.nim +++ b/src/takajopkg/splitJsonTimeline.nim @@ -1,4 +1,5 @@ -proc splitJsonTimeline(output: string = "output", quiet: bool = false, timeline: string) = +proc splitJsonTimeline(output: string = "output", quiet: bool = false, + timeline: string) = let startTime = epochTime() checkArgs(quiet, timeline, "informational") @@ -7,7 +8,11 @@ proc splitJsonTimeline(output: string = "output", quiet: bool = false, timeline: echo "This command will split a large JSONL timeline into many multiple ones based on computer name." echo "" - let totalLines = countLinesInTimeline(timeline) + var totalLines = 0 + var filePaths = getTargetExtFileLists(timeline, ".csv", true) + for timelinePath in filePaths: + totalLines += countLinesInTimeline(timelinePath) + echo "Splitting the Hayabusa JSONL timeline. Please wait." if not dirExists(output): @@ -17,7 +22,6 @@ proc splitJsonTimeline(output: string = "output", quiet: bool = false, timeline: echo "" var - inputFile = open(timeline, FileMode.fmRead) filenameSequence: seq[string] = @[] filesTable = initTable[string, File]() bar: SuruBar = initSuruBar() @@ -25,34 +29,37 @@ proc splitJsonTimeline(output: string = "output", quiet: bool = false, timeline: bar[0].total = totalLines bar.setup() - for line in lines(timeline): - inc bar - bar.update(1000000000) # refresh every second - - let jsonLineOpt = parseLine(line) - if jsonLineOpt.isNone: - continue - let jsonLine:HayabusaJson = jsonLineOpt.get() - let computerName = jsonLine.Computer - - if not filesTable.hasKey(computerName): - let filename = output & "/" & computerName & "-HayabusaResults.jsonl" - filenameSequence.add(filename) - var outputFile = open(filename, fmWrite) - filesTable[computerName] = outputFile - outputFile.write(line) - outputFile.write("\p") - flushFile(outputFile) - else: - var outputFile = filesTable[computerName] - outputFile.write(line) - outputFile.write("\p") - flushFile(outputFile) - bar.finish() - - # Close all opened files - for file in filesTable.values: - close(file) + for timelinePath in filePaths: + var inputFile = open(timelinePath, FileMode.fmRead) + + for line in lines(timeline): + inc bar + bar.update(1000000000) # refresh every second + + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine: HayabusaJson = jsonLineOpt.get() + let computerName = jsonLine.Computer + + if not filesTable.hasKey(computerName): + let filename = output & "/" & computerName & "-HayabusaResults.jsonl" + filenameSequence.add(filename) + var outputFile = open(filename, fmWrite) + filesTable[computerName] = outputFile + outputFile.write(line) + outputFile.write("\p") + flushFile(outputFile) + else: + var outputFile = filesTable[computerName] + outputFile.write(line) + outputFile.write("\p") + flushFile(outputFile) + bar.finish() + + # Close all opened files + for file in filesTable.values: + close(file) echo "" for fn in filenameSequence: @@ -60,4 +67,4 @@ proc splitJsonTimeline(output: string = "output", quiet: bool = false, timeline: echo "Saved file: " & fn & " (" & formatFileSize(fileSize) & ")" echo "" - outputElapsedTime(startTime) \ No newline at end of file + outputElapsedTime(startTime) diff --git a/src/takajopkg/stackCmdlines.nim b/src/takajopkg/stackCmdlines.nim index 03476183..450f2d66 100644 --- a/src/takajopkg/stackCmdlines.nim +++ b/src/takajopkg/stackCmdlines.nim @@ -2,31 +2,37 @@ const StackCmdlineMsg = "This command will stack executed command lines from Sys type StackCmdlineCmd* = ref object of AbstractCmd - level* :string + level*: string stack* = initTable[string, StackRecord]() - ignoreSysmon:bool - ignoreSecurity:bool + ignoreSysmon: bool + ignoreSecurity: bool -method filter*(self: StackCmdlineCmd, x: HayabusaJson):bool = - return (x.EventID == 1 and x.Channel == "Sysmon" and not self.ignoreSysmon) or (x.EventID == 4688 and x.Channel == "Sec" and not self.ignoreSecurity) +method filter*(self: StackCmdlineCmd, x: HayabusaJson): bool = + return (x.EventID == 1 and x.Channel == "Sysmon" and not self.ignoreSysmon) or + (x.EventID == 4688 and x.Channel == "Sec" and not self.ignoreSecurity) method analyze*(self: StackCmdlineCmd, x: HayabusaJson) = - let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = (x.Details["Cmdline"].getStr("N/A"), @[""]) - let (stackKey, otherColumn) = getStackKey(x) - stackResult(stackKey, self.stack, self.level, x) + let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = (x.Details[ + "Cmdline"].getStr("N/A"), @[""]) + let (stackKey, otherColumn) = getStackKey(x) + stackResult(stackKey, self.stack, self.level, x) -method resultOutput*(self: StackCmdlineCmd)= - outputResult(self, self.stack, isMinColumns=true) +method resultOutput*(self: StackCmdlineCmd) = + outputResult(self, self.stack, isMinColumns = true) -proc stackCmdlines(level: string = "low", ignoreSysmon: bool = false, ignoreSecurity: bool = false, skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc stackCmdlines(level: string = "low", ignoreSysmon: bool = false, + ignoreSecurity: bool = false, skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackCmdlineCmd( level: level, skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, - name:"stack-cmdlines", + name: "stack-cmdlines", msg: StackCmdlineMsg, ignoreSysmon: ignoreSysmon, ignoreSecurity: ignoreSecurity) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/stackComputers.nim b/src/takajopkg/stackComputers.nim index 955e1918..8309d528 100644 --- a/src/takajopkg/stackComputers.nim +++ b/src/takajopkg/stackComputers.nim @@ -2,33 +2,37 @@ const StackComputerMsg = "This command will stack the Computer (default) or SrcC type StackComputersCmd* = ref object of AbstractCmd - level* :string + level*: string stack* = initTable[string, StackRecord]() - sourceComputers:bool + sourceComputers: bool -method filter*(self: StackComputersCmd, x: HayabusaJson):bool = - return true +method filter*(self: StackComputersCmd, x: HayabusaJson): bool = + return true -method analyze*(self: StackComputersCmd, x: HayabusaJson)= - let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = - var stackKey = x.Computer - if self.sourceComputers: - stackKey = getJsonValue(x.Details, @["SrcComp"]) - return (stackKey, @[""]) - let (stackKey, otherColumn) = getStackKey(x) - stackResult(stackKey, self.stack, self.level, x) +method analyze*(self: StackComputersCmd, x: HayabusaJson) = + let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = + var stackKey = x.Computer + if self.sourceComputers: + stackKey = getJsonValue(x.Details, @["SrcComp"]) + return (stackKey, @[""]) + let (stackKey, otherColumn) = getStackKey(x) + stackResult(stackKey, self.stack, self.level, x) -method resultOutput*(self: StackComputersCmd)= - outputResult(self, self.stack, isMinColumns=true) +method resultOutput*(self: StackComputersCmd) = + outputResult(self, self.stack, isMinColumns = true) -proc stackComputers(level: string = "informational", sourceComputers: bool = false, skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc stackComputers(level: string = "informational", + sourceComputers: bool = false, skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackComputersCmd( level: level, skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, name: "stack-computers", msg: StackComputerMsg, sourceComputers: sourceComputers) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/stackDNS.nim b/src/takajopkg/stackDNS.nim index e1905fcd..23478bbf 100644 --- a/src/takajopkg/stackDNS.nim +++ b/src/takajopkg/stackDNS.nim @@ -2,33 +2,36 @@ const StackDNSMsg = "This command will stack the DNS queries and responses from type StackDNSCmd* = ref object of AbstractCmd - level* :string + level*: string header* = @["Image", "Query", "Result"] stack* = initTable[string, StackRecord]() -method filter*(self: StackDNSCmd, x: HayabusaJson):bool = - return x.EventID == 22 and x.Channel == "Sysmon" +method filter*(self: StackDNSCmd, x: HayabusaJson): bool = + return x.EventID == 22 and x.Channel == "Sysmon" -method analyze*(self: StackDNSCmd, x: HayabusaJson)= - let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = - let pro = x.Details["Proc"].getStr("N/A") - let que = x.Details["Query"].getStr("N/A") - let res = x.Details["Result"].getStr("N/A") - let stackKey = pro & "->" & que & "->" & res - return (stackKey, @[pro, que, res]) - let (stackKey, otherColumn) = getStackKey(x) - stackResult(stackKey, self.stack, self.level, x, otherColumn) +method analyze*(self: StackDNSCmd, x: HayabusaJson) = + let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = + let pro = x.Details["Proc"].getStr("N/A") + let que = x.Details["Query"].getStr("N/A") + let res = x.Details["Result"].getStr("N/A") + let stackKey = pro & "->" & que & "->" & res + return (stackKey, @[pro, que, res]) + let (stackKey, otherColumn) = getStackKey(x) + stackResult(stackKey, self.stack, self.level, x, otherColumn) -method resultOutput*(self: StackDNSCmd)= - outputResult(self, self.stack, self.header) +method resultOutput*(self: StackDNSCmd) = + outputResult(self, self.stack, self.header) -proc stackDNS(level: string = "informational", skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc stackDNS(level: string = "informational", skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackDNSCmd( level: level, skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, name: "stack-dns", msg: StackDNSMsg) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/stackIpAddresses.nim b/src/takajopkg/stackIpAddresses.nim index 7fa22216..831fd999 100644 --- a/src/takajopkg/stackIpAddresses.nim +++ b/src/takajopkg/stackIpAddresses.nim @@ -2,32 +2,37 @@ const StackIpAddressesMsg = "This command will stack the SrcIP (default) or TgtI type StackIpAddressesCmd* = ref object of AbstractCmd - level* :string + level*: string stack* = initTable[string, StackRecord]() - targetIpAddresses:bool + targetIpAddresses: bool -method filter*(self: StackIpAddressesCmd, x: HayabusaJson):bool = - let key = if self.targetIpAddresses: "TgtIP" else: "SrcIP" - let ip = getJsonValue(x.Details, @[key]) - return ip != "127.0.0.1" and ip != "::1" +method filter*(self: StackIpAddressesCmd, x: HayabusaJson): bool = + let key = if self.targetIpAddresses: "TgtIP" else: "SrcIP" + let ip = getJsonValue(x.Details, @[key]) + return ip != "127.0.0.1" and ip != "::1" -method analyze*(self: StackIpAddressesCmd, x: HayabusaJson)= - let key = if self.targetIpAddresses: "TgtIP" else: "SrcIP" - let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = (getJsonValue(x.Details, @[key]), @[""]) - let (stackKey, otherColumn) = getStackKey(x) - stackResult(stackKey, self.stack, self.level, x) +method analyze*(self: StackIpAddressesCmd, x: HayabusaJson) = + let key = if self.targetIpAddresses: "TgtIP" else: "SrcIP" + let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = ( + getJsonValue(x.Details, @[key]), @[""]) + let (stackKey, otherColumn) = getStackKey(x) + stackResult(stackKey, self.stack, self.level, x) -method resultOutput*(self: StackIpAddressesCmd)= - outputResult(self, self.stack, isMinColumns=true) +method resultOutput*(self: StackIpAddressesCmd) = + outputResult(self, self.stack, isMinColumns = true) -proc stackIpAddresses(level: string = "informational", targetIpAddresses: bool = false, skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc stackIpAddresses(level: string = "informational", + targetIpAddresses: bool = false, skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackIpAddressesCmd( level: level, skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, name: "stack-ip-addresses", msg: StackIpAddressesMsg, targetIpAddresses: targetIpAddresses) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/stackLogons.nim b/src/takajopkg/stackLogons.nim index 56e43081..e8fcc645 100644 --- a/src/takajopkg/stackLogons.nim +++ b/src/takajopkg/stackLogons.nim @@ -5,79 +5,83 @@ const StackLogonsMsg = "This command will stack logons based on target user, tar type StackLogonsCmd* = ref object of AbstractCmd - seqOfStrings*: seq[string] - uniqueLogons = 0 - localSrcIpAddresses:bool + seqOfStrings*: seq[string] + uniqueLogons = 0 + localSrcIpAddresses: bool -method filter*(self: StackLogonsCmd, x: HayabusaJson):bool = - return isEID_4624(x.RuleTitle) +method filter*(self: StackLogonsCmd, x: HayabusaJson): bool = + return isEID_4624(x.RuleTitle) method analyze*(self: StackLogonsCmd, x: HayabusaJson) = - let - tgtUser = getJsonValue(x.Details, @["TgtUser"]) - tgtComp = x.Computer - logonType = getJsonValue(x.Details, @["Type"]) - srcIP = getJsonValue(x.Details, @["SrcIP"]) - srcComp = getJsonValue(x.Details, @["SrcComp"]) + let + tgtUser = getJsonValue(x.Details, @["TgtUser"]) + tgtComp = x.Computer + logonType = getJsonValue(x.Details, @["Type"]) + srcIP = getJsonValue(x.Details, @["SrcIP"]) + srcComp = getJsonValue(x.Details, @["SrcComp"]) - if not self.localSrcIpAddresses and isLocalIP(srcIP): - discard - else: - self.seqOfStrings.add(tgtUser & "," & tgtComp & "," & logonType & "," & srcIP & "," & srcComp) + if not self.localSrcIpAddresses and isLocalIP(srcIP): + discard + else: + self.seqOfStrings.add(tgtUser & "," & tgtComp & "," & logonType & "," & + srcIP & "," & srcComp) method resultOutput*(self: StackLogonsCmd) = - var countsTable: Table[string, int] = initTable[string, int]() + var countsTable: Table[string, int] = initTable[string, int]() - # Add a count for each time the unique string was found - for string in self.seqOfStrings: - if not countsTable.hasKey(string): - countsTable[string] = 0 - countsTable[string] += 1 + # Add a count for each time the unique string was found + for string in self.seqOfStrings: + if not countsTable.hasKey(string): + countsTable[string] = 0 + countsTable[string] += 1 - # Create a sequence of pairs from the Table - var seqOfPairs: seq[(string, int)] = @[] - for key, val in countsTable: - seqOfPairs.add((key, val)) + # Create a sequence of pairs from the Table + var seqOfPairs: seq[(string, int)] = @[] + for key, val in countsTable: + seqOfPairs.add((key, val)) - # Sort the sequence in descending order based on the count - seqOfPairs.sort(proc (x, y: (string, int)): int = y[1] - x[1]) + # Sort the sequence in descending order based on the count + seqOfPairs.sort(proc (x, y: (string, int)): int = y[1] - x[1]) - # Print results to screen - var outputFileSize = 0 - if self.output == "": - # Print the sorted counts with unique strings - for (string, count) in seqOfPairs: - inc self.uniqueLogons - var commaDelimitedStr = $count & "," & string - commaDelimitedStr = replace(commaDelimitedStr, ",", " | ") - echo commaDelimitedStr - # Save to CSV file - else: - let outputFile = open(self.output, fmWrite) - # Write headers - writeLine(outputFile, "Count,TgtUser,TgtComp,LogonType,SrcIP,SrcComp") + # Print results to screen + var outputFileSize = 0 + if self.output == "": + # Print the sorted counts with unique strings + for (string, count) in seqOfPairs: + inc self.uniqueLogons + var commaDelimitedStr = $count & "," & string + commaDelimitedStr = replace(commaDelimitedStr, ",", " | ") + echo commaDelimitedStr + # Save to CSV file + else: + let outputFile = open(self.output, fmWrite) + # Write headers + writeLine(outputFile, "Count,TgtUser,TgtComp,LogonType,SrcIP,SrcComp") - # Write results - for (string, count) in seqOfPairs: - inc self.uniqueLogons - writeLine(outputFile, $count & "," & string) - outputFileSize = getFileSize(outputFile) - close(outputFile) - let results = "Unique logons: " & $self.uniqueLogons - let savedFiles = self.output & " (" & formatFileSize(outputFileSize) & ")" - if self.displayTable: - echo "" - echo results - echo "Saved file: " & savedFiles - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) + # Write results + for (string, count) in seqOfPairs: + inc self.uniqueLogons + writeLine(outputFile, $count & "," & string) + outputFileSize = getFileSize(outputFile) + close(outputFile) + let results = "Unique logons: " & $self.uniqueLogons + let savedFiles = self.output & " (" & formatFileSize(outputFileSize) & ")" + if self.displayTable: + echo "" + echo results + echo "Saved file: " & savedFiles + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc stackLogons(localSrcIpAddresses = false, skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, "informational") +proc stackLogons(localSrcIpAddresses = false, skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, "informational") + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackLogonsCmd( skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, - name:"stack-logons", + name: "stack-logons", msg: StackLogonsMsg, localSrcIpAddresses: localSrcIpAddresses) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/stackProcesses.nim b/src/takajopkg/stackProcesses.nim index 46d5e4cc..a8ccc45c 100644 --- a/src/takajopkg/stackProcesses.nim +++ b/src/takajopkg/stackProcesses.nim @@ -2,31 +2,37 @@ const StackProcessesMsg = "This command will stack the executed processes from S type StackProcessesCmd* = ref object of AbstractCmd - level* :string + level*: string stack* = initTable[string, StackRecord]() - ignoreSysmon:bool - ignoreSecurity:bool + ignoreSysmon: bool + ignoreSecurity: bool -method filter*(self: StackProcessesCmd, x: HayabusaJson):bool = - return (x.EventID == 1 and not self.ignoreSysmon and x.Channel == "Sysmon") or (x.EventID == 4688 and not self.ignoreSecurity and x.Channel == "Sec") +method filter*(self: StackProcessesCmd, x: HayabusaJson): bool = + return (x.EventID == 1 and not self.ignoreSysmon and x.Channel == "Sysmon") or + (x.EventID == 4688 and not self.ignoreSecurity and x.Channel == "Sec") -method analyze*(self: StackProcessesCmd, x: HayabusaJson)= - let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = (x.Details["Proc"].getStr("N/A"), @[""]) - let (stackKey, otherColumn) = getStackKey(x) - stackResult(stackKey, self.stack, self.level, x) +method analyze*(self: StackProcessesCmd, x: HayabusaJson) = + let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = (x.Details[ + "Proc"].getStr("N/A"), @[""]) + let (stackKey, otherColumn) = getStackKey(x) + stackResult(stackKey, self.stack, self.level, x) method resultOutput*(self: StackProcessesCmd) = - outputResult(self, self.stack, isMinColumns=true) + outputResult(self, self.stack, isMinColumns = true) -proc stackProcesses(level: string = "low", ignoreSysmon: bool = false, ignoreSecurity: bool = false, skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc stackProcesses(level: string = "low", ignoreSysmon: bool = false, + ignoreSecurity: bool = false, skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackProcessesCmd( level: level, skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, name: "stack-processes", msg: StackProcessesMsg, ignoreSysmon: ignoreSysmon, ignoreSecurity: ignoreSecurity) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/stackServices.nim b/src/takajopkg/stackServices.nim index 746e1a5e..df4970be 100644 --- a/src/takajopkg/stackServices.nim +++ b/src/takajopkg/stackServices.nim @@ -2,36 +2,41 @@ const StackServicesMsg = "This command will stack the service names and paths fr type StackServicesCmd* = ref object of AbstractCmd - level* :string + level*: string header* = @["ServiceName", "Path"] stack* = initTable[string, StackRecord]() - ignoreSystem:bool - ignoreSecurity:bool + ignoreSystem: bool + ignoreSecurity: bool -method filter*(self: StackServicesCmd, x: HayabusaJson):bool = - return (x.EventID == 7045 and not self.ignoreSystem and x.Channel == "Sys") or (x.EventID == 4697 and not self.ignoreSecurity and x.Channel == "Sec") +method filter*(self: StackServicesCmd, x: HayabusaJson): bool = + return (x.EventID == 7045 and not self.ignoreSystem and x.Channel == "Sys") or + (x.EventID == 4697 and not self.ignoreSecurity and x.Channel == "Sec") -method analyze*(self: StackServicesCmd, x: HayabusaJson)= - let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = - let svc = x.Details["Svc"].getStr("N/A") - let pat = x.Details["Path"].getStr("N/A") - let stackKey = svc & " -> " & pat - return (stackKey, @[svc, pat]) - let (stackKey, otherColumn) = getStackKey(x) - stackResult(stackKey, self.stack, self.level, x, otherColumn=otherColumn) +method analyze*(self: StackServicesCmd, x: HayabusaJson) = + let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = + let svc = x.Details["Svc"].getStr("N/A") + let pat = x.Details["Path"].getStr("N/A") + let stackKey = svc & " -> " & pat + return (stackKey, @[svc, pat]) + let (stackKey, otherColumn) = getStackKey(x) + stackResult(stackKey, self.stack, self.level, x, otherColumn = otherColumn) -method resultOutput*(self: StackServicesCmd)= - outputResult(self, self.stack, self.header) +method resultOutput*(self: StackServicesCmd) = + outputResult(self, self.stack, self.header) -proc stackServices(level: string = "informational", ignoreSystem: bool = false, ignoreSecurity: bool = false, skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc stackServices(level: string = "informational", ignoreSystem: bool = false, + ignoreSecurity: bool = false, skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackServicesCmd( level: level, skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, name: "stack-services", msg: StackServicesMsg, ignoreSystem: ignoreSystem, ignoreSecurity: ignoreSecurity) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/stackTasks.nim b/src/takajopkg/stackTasks.nim index bdb6bf1a..3dd61c8c 100644 --- a/src/takajopkg/stackTasks.nim +++ b/src/takajopkg/stackTasks.nim @@ -2,51 +2,56 @@ const StackTasksMsg = "This command will stack new scheduled tasks from Security type StackTasksCmd* = ref object of AbstractCmd - level* :string + level*: string header* = @["TaskName", "Command", "Arguments"] stack* = initTable[string, StackRecord]() - ignoreSysmon:bool - ignoreSecurity:bool + ignoreSysmon: bool + ignoreSecurity: bool -method filter*(self: StackTasksCmd, x: HayabusaJson):bool = - return x.EventID == 4698 and x.Channel == "Sec" +method filter*(self: StackTasksCmd, x: HayabusaJson): bool = + return x.EventID == 4698 and x.Channel == "Sec" proc decodeEntity*(txt: string): string = - return txt.replace("&","&").replace("<","<").replace(">",">").replace(""","\"").replace("'","'") + return txt.replace("&", "&").replace("<", "<").replace(">", + ">").replace(""", "\"").replace("'", "'") -method analyze*(self: StackTasksCmd, x: HayabusaJson)= - let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = - let user = x.Details["User"].getStr("N/A") - let name = x.Details["Name"].getStr("N/A") - let content = x.Details["Content"].getStr("N/A").replace("\\r\\n", "") - let node = parseXml(content) - let commands = node.findAll("Command") - var command = "" - var args = "" - if len(commands) > 0: - command = $commands[0] - command = command.replace("", "").replace("","") - let arguments = node.findAll("Arguments") - if len(arguments) > 0: - args = $arguments[0] - args = args.replace("", "").replace("","") - let stackKey = user & " -> " & name & " -> " & decodeEntity(command & " " & args) - return (stackKey, @[name, command, args]) - let (stackKey, otherColumn) = getStackKey(x) - stackResult(stackKey, self.stack, self.level, x, otherColumn=otherColumn) +method analyze*(self: StackTasksCmd, x: HayabusaJson) = + let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = + let user = x.Details["User"].getStr("N/A") + let name = x.Details["Name"].getStr("N/A") + let content = x.Details["Content"].getStr("N/A").replace("\\r\\n", "") + let node = parseXml(content) + let commands = node.findAll("Command") + var command = "" + var args = "" + if len(commands) > 0: + command = $commands[0] + command = command.replace("", "").replace("", "") + let arguments = node.findAll("Arguments") + if len(arguments) > 0: + args = $arguments[0] + args = args.replace("", "").replace("", "") + let stackKey = user & " -> " & name & " -> " & decodeEntity(command & " " & args) + return (stackKey, @[name, command, args]) + let (stackKey, otherColumn) = getStackKey(x) + stackResult(stackKey, self.stack, self.level, x, otherColumn = otherColumn) -method resultOutput*(self: StackTasksCmd)= - outputResult(self, self.stack, self.header) +method resultOutput*(self: StackTasksCmd) = + outputResult(self, self.stack, self.header) -proc stackTasks(level: string = "informational", ignoreSysmon: bool = false, ignoreSecurity: bool = false, skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc stackTasks(level: string = "informational", ignoreSysmon: bool = false, + ignoreSecurity: bool = false, skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackTasksCmd( level: level, skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, name: "stack-tasks", msg: StackTasksMsg, ignoreSysmon: ignoreSysmon, ignoreSecurity: ignoreSecurity) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/stackUsers.nim b/src/takajopkg/stackUsers.nim index 0c41ec76..f86f8cb5 100644 --- a/src/takajopkg/stackUsers.nim +++ b/src/takajopkg/stackUsers.nim @@ -1,12 +1,18 @@ # 以下MSドキュメントを参考にシステムアカウントを列挙 # https://learn.microsoft.com/ja-jp/sql/database-engine/configure-windows/configure-windows-service-accounts-and-permissions?view=sql-server-ver16 const SYSTEM_ACCOUNTS = toHashSet([ - "NT AUTHORITY\\LOCAL SERVICE","NT AUTHORITY\\NETWORK SERVICE","NT AUTHORITY\\SYSTEM","BUILTIN\\Administrators", - "NT-AUTORITÄT\\LOKALER DIENST","NT-AUTORITÄT\\NETZWERKDIENST","NT-AUTORITÄT\\SYSTEM","VORDEFINIERT\\Administratoren", - "AUTORITE NT\\SERVICE LOCAL","AUTORITE NT\\SERVICE RÉSEAU","AUTORITE NT\\SYSTEM","BUILTIN\\Administrators", - "NT AUTHORITY\\SERVIZIO LOCALE","NT AUTHORITY\\SERVIZIO DI RETE","NT AUTHORITY\\SYSTEM","BUILTIN\\Administrators", - "NT AUTHORITY\\SERVICIO LOC","NT AUTHORITY\\SERVICIO DE RED","NT AUTHORITY\\SYSTEM","BUILTIN\\Administradores", - "NT AUTHORITY\\LOCAL SERVICE","NT AUTHORITY\\NETWORK SERVICE","NT AUTHORITY\\СИСТЕМА","BUILTIN\\Администраторы", + "NT AUTHORITY\\LOCAL SERVICE", "NT AUTHORITY\\NETWORK SERVICE", + "NT AUTHORITY\\SYSTEM", "BUILTIN\\Administrators", + "NT-AUTORITÄT\\LOKALER DIENST", "NT-AUTORITÄT\\NETZWERKDIENST", + "NT-AUTORITÄT\\SYSTEM", "VORDEFINIERT\\Administratoren", + "AUTORITE NT\\SERVICE LOCAL", "AUTORITE NT\\SERVICE RÉSEAU", + "AUTORITE NT\\SYSTEM", "BUILTIN\\Administrators", + "NT AUTHORITY\\SERVIZIO LOCALE", "NT AUTHORITY\\SERVIZIO DI RETE", + "NT AUTHORITY\\SYSTEM", "BUILTIN\\Administrators", + "NT AUTHORITY\\SERVICIO LOC", "NT AUTHORITY\\SERVICIO DE RED", + "NT AUTHORITY\\SYSTEM", "BUILTIN\\Administradores", + "NT AUTHORITY\\LOCAL SERVICE", "NT AUTHORITY\\NETWORK SERVICE", + "NT AUTHORITY\\СИСТЕМА", "BUILTIN\\Администраторы", "Window Manager\\DWM-1", "LOCAL SERVICE", "IIS APPPOOL\\DefaultAppPool" @@ -16,45 +22,50 @@ const StackUsersMsg = "This command will stack the TgtUser (default) or SrcUser type StackUsersCmd* = ref object of AbstractCmd - level* :string + level*: string stack* = initTable[string, StackRecord]() - sourceUsers:bool - filterComputerAccounts:bool - filterSystemAccounts:bool + sourceUsers: bool + filterComputerAccounts: bool + filterSystemAccounts: bool -method filter*(self: StackUsersCmd, x: HayabusaJson):bool = - return true +method filter*(self: StackUsersCmd, x: HayabusaJson): bool = + return true -method analyze*(self: StackUsersCmd, x: HayabusaJson)= - let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = - var stackKey = getJsonValue(x.Details, @["User"]) - if stackKey.len() == 0: - let key = if self.sourceUsers: "SrcUser" else: "TgtUser" - stackKey = getJsonValue(x.Details, @[key]) - if self.filterComputerAccounts and stackKey.endsWith("$"): - stackKey = "" - if self.filterSystemAccounts: - for excludeAccount in SYSTEM_ACCOUNTS: - if cmpIgnoreCase(stackKey, excludeAccount) == 0: - stackKey = "" - continue - return (stackKey, @[""]) - let (stackKey, otherColumn) = getStackKey(x) - stackResult(stackKey, self.stack, self.level, x) +method analyze*(self: StackUsersCmd, x: HayabusaJson) = + let getStackKey = proc(x: HayabusaJson): (string, seq[string]) = + var stackKey = getJsonValue(x.Details, @["User"]) + if stackKey.len() == 0: + let key = if self.sourceUsers: "SrcUser" else: "TgtUser" + stackKey = getJsonValue(x.Details, @[key]) + if self.filterComputerAccounts and stackKey.endsWith("$"): + stackKey = "" + if self.filterSystemAccounts: + for excludeAccount in SYSTEM_ACCOUNTS: + if cmpIgnoreCase(stackKey, excludeAccount) == 0: + stackKey = "" + continue + return (stackKey, @[""]) + let (stackKey, otherColumn) = getStackKey(x) + stackResult(stackKey, self.stack, self.level, x) method resultOutput*(self: StackUsersCmd) = - outputResult(self, self.stack, isMinColumns=true) + outputResult(self, self.stack, isMinColumns = true) -proc stackUsers(level: string = "informational", sourceUsers: bool = false, filterComputerAccounts: bool = true, filterSystemAccounts: bool = true, skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, level) +proc stackUsers(level: string = "informational", sourceUsers: bool = false, + filterComputerAccounts: bool = true, filterSystemAccounts: bool = true, + skipProgressBar: bool = false, output: string = "", quiet: bool = false, + timeline: string) = + checkArgs(quiet, timeline, level) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = StackUsersCmd( level: level, skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, name: "stack-users", msg: StackUsersMsg, sourceUsers: sourceUsers, filterSystemAccounts: filterSystemAccounts, filterComputerAccounts: filterComputerAccounts) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/sysmonProcessTree.nim b/src/takajopkg/sysmonProcessTree.nim index 19e55192..efce803f 100644 --- a/src/takajopkg/sysmonProcessTree.nim +++ b/src/takajopkg/sysmonProcessTree.nim @@ -52,10 +52,12 @@ proc moveProcessObjectToChild(mvSource: processObject, ## Procedure for moving a process object to a child process for i, childProc in target.children: let s = output.children.len - if childProc.processGUID == mvSource.parentProcessGUID and output.children.len - 1 >= i: + if childProc.processGUID == mvSource.parentProcessGUID and + output.children.len - 1 >= i: # Added to a separate table because assertion errors occur when the number of elements changes during iteration output.children[i].children.add(mvSource) - output.children[i].children = sorted(deduplicate(output.children[i].children), cmpTimeStamp) + output.children[i].children = sorted(deduplicate(output.children[ + i].children), cmpTimeStamp) return else: if output.children.len - 1 < i: @@ -114,40 +116,41 @@ proc sysmonProcessTree(output: string = "", processGuid: string, var oldestProc: processObject var passGuid = initHashSet[string]() passGuid.incl(processGuid) - - for line in lines(timeline): - let - jsonLine = parseJson(line) - timeStamp = jsonLine["Timestamp"].getStr("N/A") - channel = jsonLine["Channel"].getStr("N/A") - eventId = jsonLine["EventID"].getInt(0) - ruleTitle = jsonLine["RuleTitle"].getStr("N/A") - - # Found a Sysmon 1 process creation event. - # This assumes info level events are enabled and there won't be more than one Sysmon 1 event for a process. - if channel == "Sysmon" and eventId == 1 and (ruleTitle == "Proc Exec" or - ruleTitle == "Proc Exec (Sysmon Alert)"): - if jsonLine["Details"]["PGUID"].getStr("N/A") in passGuid or - jsonLine["Details"]["ParentPGUID"].getStr("N/A") in passGuid: - let obj = createProcessObj(jsonLine, false) - if procFoundCount == 0: - let parentObj = createProcessObj(jsonLine, true) - stockedProcObjTbl[parentObj.processGUID] = parentObj - stockedProcObjTbl[parentObj.processGUID].children.add(obj) - parentPGUIDTbl[parentObj.processGUID] = obj.processGUID - oldestProc = parentObj - passGuid.incl(obj.processGUID) - passGuid.incl(obj.parentProcessGUID) - # Link child processes to their parents - stockedProcObjTbl[obj.processGUID] = obj - if obj.parentProcessGUID == processGUID: - stockedProcObjTbl[processGUID].children.add(obj) - elif obj.parentProcessGUID in stockedProcObjTbl: - stockedProcObjTbl[obj.parentProcessGUID].children.add(obj) - parentPGUIDTbl[obj.parentProcessGUID] = obj.processGUID - inc procFoundCount - elif procFoundCount == 0: - parentsProcStocks.add(line) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: + for line in lines(timelinePath): + let + jsonLine = parseJson(line) + timeStamp = jsonLine["Timestamp"].getStr("N/A") + channel = jsonLine["Channel"].getStr("N/A") + eventId = jsonLine["EventID"].getInt(0) + ruleTitle = jsonLine["RuleTitle"].getStr("N/A") + + # Found a Sysmon 1 process creation event. + # This assumes info level events are enabled and there won't be more than one Sysmon 1 event for a process. + if channel == "Sysmon" and eventId == 1 and (ruleTitle == + "Proc Exec" or ruleTitle == "Proc Exec (Sysmon Alert)"): + if jsonLine["Details"]["PGUID"].getStr("N/A") in passGuid or + jsonLine["Details"]["ParentPGUID"].getStr("N/A") in passGuid: + let obj = createProcessObj(jsonLine, false) + if procFoundCount == 0: + let parentObj = createProcessObj(jsonLine, true) + stockedProcObjTbl[parentObj.processGUID] = parentObj + stockedProcObjTbl[parentObj.processGUID].children.add(obj) + parentPGUIDTbl[parentObj.processGUID] = obj.processGUID + oldestProc = parentObj + passGuid.incl(obj.processGUID) + passGuid.incl(obj.parentProcessGUID) + # Link child processes to their parents + stockedProcObjTbl[obj.processGUID] = obj + if obj.parentProcessGUID == processGUID: + stockedProcObjTbl[processGUID].children.add(obj) + elif obj.parentProcessGUID in stockedProcObjTbl: + stockedProcObjTbl[obj.parentProcessGUID].children.add(obj) + parentPGUIDTbl[obj.parentProcessGUID] = obj.processGUID + inc procFoundCount + elif procFoundCount == 0: + parentsProcStocks.add(line) if procFoundCount == 0: echo "The process was not found." @@ -212,4 +215,4 @@ proc sysmonProcessTree(output: string = "", processGuid: string, else: echo line echo "" - outputElapsedTime(startTime) \ No newline at end of file + outputElapsedTime(startTime) diff --git a/src/takajopkg/timelineLogon.nim b/src/takajopkg/timelineLogon.nim index 09ee3057..1275c11b 100644 --- a/src/takajopkg/timelineLogon.nim +++ b/src/takajopkg/timelineLogon.nim @@ -6,23 +6,25 @@ Logoff events can be outputted on separate lines with -l, --outputLogoffEvents. Admin logon events can be outputted on separate lines with -a, --outputAdminLogonEvents.""" type - TimelineLogonCmd* = ref object of AbstractCmd - seqOfResultsTables*: seq[TableRef[string, string]] # Sequences are immutable so need to create a sequence of pointers to tables so we can update ["ElapsedTime"] - seqOfLogoffEventTables*: seq[Table[string, string]] # This sequence can be immutable - logoffEvents*: Table[string, string] = initTable[string, string]() - adminLogonEvents*: Table[string, string] = initTable[string, string]() - EID_4624_count* = 0 # Successful logon - EID_4625_count* = 0 # Failed logon - EID_4634_count* = 0 # Logoff - EID_4647_count* = 0 # User initiated logoff - EID_4648_count* = 0 # Explicit logon - EID_4672_count* = 0 # Admin logon - calculateElapsedTime* :bool - outputLogoffEvents*: bool - outputAdminLogonEvents*: bool + TimelineLogonCmd* = ref object of AbstractCmd + seqOfResultsTables*: seq[TableRef[string, + string]] # Sequences are immutable so need to create a sequence of pointers to tables so we can update ["ElapsedTime"] + seqOfLogoffEventTables*: seq[Table[string, string]] # This sequence can be immutable + logoffEvents*: Table[string, string] = initTable[string, string]() + adminLogonEvents*: Table[string, string] = initTable[string, string]() + EID_4624_count* = 0 # Successful logon + EID_4625_count* = 0 # Failed logon + EID_4634_count* = 0 # Logoff + EID_4647_count* = 0 # User initiated logoff + EID_4648_count* = 0 # Explicit logon + EID_4672_count* = 0 # Admin logon + calculateElapsedTime*: bool + outputLogoffEvents*: bool + outputAdminLogonEvents*: bool -method filter*(self: TimelineLogonCmd, x: HayabusaJson):bool = - return x.EventId == 4624 or x.EventId == 4625 or x.EventId == 4634 or x.EventId == 4647 or x.EventId == 4648 or x.EventId == 4672 +method filter*(self: TimelineLogonCmd, x: HayabusaJson): bool = + return x.EventId == 4624 or x.EventId == 4625 or x.EventId == 4634 or + x.EventId == 4647 or x.EventId == 4648 or x.EventId == 4672 method analyze*(self: TimelineLogonCmd, x: HayabusaJson) = let ruleTitle = x.RuleTitle @@ -37,7 +39,8 @@ method analyze*(self: TimelineLogonCmd, x: HayabusaJson) = singleResultTable["EventID"] = "4624" let details = jsonLine.Details let extraFieldInfo = jsonLine.ExtraFieldInfo - let logonType = details.extractStr("Type") # Will be Int if field mapping is turned off + let logonType = details.extractStr( + "Type") # Will be Int if field mapping is turned off singleResultTable["Type"] = logonType # Needs to be logonNumberToString(logonType) if data field mapping is turned off. TODO singleResultTable["Auth"] = extraFieldInfo.extractStr("AuthenticationPackageName") singleResultTable["TargetComputer"] = jsonLine.Computer @@ -77,9 +80,11 @@ method analyze*(self: TimelineLogonCmd, x: HayabusaJson) = singleResultTable["SourceIP"] = details.extractStr("SrcIP") singleResultTable["Process"] = details.extractStr("Proc") singleResultTable["SourceComputer"] = details.extractStr("SrcComp") - singleResultTable["TargetUserSID"] = extraFieldInfo.extractStr("TargetUserSid") # Don't output as it is always S-1-0-0 + singleResultTable["TargetUserSID"] = extraFieldInfo.extractStr( + "TargetUserSid") # Don't output as it is always S-1-0-0 singleResultTable["TargetDomainName"] = extraFieldInfo.extractStr("TargetDomainName") - singleResultTable["FailureReason"] = logonFailureReason(extraFieldInfo.extractStr("SubStatus")) + singleResultTable["FailureReason"] = logonFailureReason( + extraFieldInfo.extractStr("SubStatus")) self.seqOfResultsTables.add(singleResultTable) #EID 4634 Logoff @@ -88,7 +93,8 @@ method analyze*(self: TimelineLogonCmd, x: HayabusaJson) = # If we want to calculate ElapsedTime if self.calculateElapsedTime: # Create the key in the format of LID:Computer:User with a value of the timestamp - let key = jsonLine.Details["LID"].getStr() & ":" & jsonLine.Computer & ":" & jsonLine.Details["User"].getStr() + let key = jsonLine.Details["LID"].getStr() & ":" & + jsonLine.Computer & ":" & jsonLine.Details["User"].getStr() let logoffTime = jsonLine.Timestamp self.logoffEvents[key] = logoffTime if self.outputLogoffEvents: @@ -109,7 +115,8 @@ method analyze*(self: TimelineLogonCmd, x: HayabusaJson) = # If we want to calculate ElapsedTime if self.calculateElapsedTime: # Create the key in the format of LID:Computer:User with a value of the timestamp - let key = jsonLine.Details["LID"].getStr() & ":" & jsonLine.Computer & ":" & jsonLine.Details["User"].getStr() + let key = jsonLine.Details["LID"].getStr() & ":" & + jsonLine.Computer & ":" & jsonLine.Details["User"].getStr() let logoffTime = jsonLine.Timestamp self.logoffEvents[key] = logoffTime if self.outputLogoffEvents: @@ -150,7 +157,8 @@ method analyze*(self: TimelineLogonCmd, x: HayabusaJson) = # The timing will be very close to the 4624 log so I am checking if the Computer, LID and TgtUser are the same and then if the two events happened within 10 seconds. if ruleTitle == "Admin Logon": inc self.EID_4672_count - let key = jsonLine.Details["LID"].getStr() & ":" & jsonLine.Computer & ":" & jsonLine.Details["TgtUser"].getStr() + let key = jsonLine.Details["LID"].getStr() & ":" & jsonLine.Computer & + ":" & jsonLine.Details["TgtUser"].getStr() let adminLogonTime = jsonLine.Timestamp self.adminLogonEvents[key] = adminLogonTime if self.outputAdminLogonEvents: @@ -179,14 +187,20 @@ method resultOutput*(self: TimelineLogonCmd) = var logoffTime = "" var logonTime = tableOfResults["Timestamp"] - let key = tableOfResults["LID"] & ":" & tableOfResults["TargetComputer"] & ":" & tableOfResults["TargetUser"] + let key = tableOfResults["LID"] & ":" & tableOfResults[ + "TargetComputer"] & ":" & tableOfResults["TargetUser"] if self.logoffEvents.hasKey(key): logoffTime = self.logoffEvents[key] tableOfResults[]["LogoffTime"] = logoffTime - logonTime = if logonTime.endsWith("Z"): logonTime.replace("Z","") else: logonTime[0 ..< logonTime.len - 7] - logoffTime = if logoffTime.endsWith("Z"): logoffTime.replace("Z","") else: logoffTime[0 ..< logofftime.len - 7] - let parsedLogoffTime = parse(padString(logoffTime, '0', timeFormat), timeFormat) - let parsedLogonTime = parse(padString(logonTime, '0', timeFormat), timeFormat) + logonTime = if logonTime.endsWith("Z"): logonTime.replace( + "Z", "") else: logonTime[0 ..< logonTime.len - 7] + logoffTime = if logoffTime.endsWith( + "Z"): logoffTime.replace("Z", "") else: logoffTime[ + 0 ..< logofftime.len - 7] + let parsedLogoffTime = parse(padString(logoffTime, '0', + timeFormat), timeFormat) + let parsedLogonTime = parse(padString(logonTime, '0', + timeFormat), timeFormat) let duration = parsedLogoffTime - parsedLogonTime tableOfResults[]["ElapsedTime"] = formatDuration(duration) else: @@ -196,39 +210,61 @@ method resultOutput*(self: TimelineLogonCmd) = for tableOfResults in self.seqOfResultsTables: if tableOfResults["EventID"] == "4624": var logonTime = tableOfResults["Timestamp"] - logonTime = if logonTime.endsWith("Z"): logonTime.replace("Z","") else: logonTime[0 ..< logonTime.len - 7] # Remove the timezone + logonTime = if logonTime.endsWith("Z"): logonTime.replace("Z", + "") else: logonTime[0 ..< logonTime.len - 7] # Remove the timezone #echo "4624 logon time: " & logonTime - let key = tableOfResults["LID"] & ":" & tableOfResults["TargetComputer"] & ":" & tableOfResults["TargetUser"] + let key = tableOfResults["LID"] & ":" & tableOfResults[ + "TargetComputer"] & ":" & tableOfResults["TargetUser"] if self.adminLogonEvents.hasKey(key): var adminLogonTime = self.adminLogonEvents[key] - adminLogonTime = if adminLogonTime.endsWith("Z"): adminLogonTime.replace("Z","") else: adminLogonTime[0 ..< adminLogonTime.len - 7] # Remove the timezone - let parsed_4624_logonTime = parse(padString(logonTime, '0', timeFormat), timeFormat) - let parsed_4672_logonTime = parse(padString(adminLogonTime, '0', timeFormat), timeFormat) + adminLogonTime = if adminLogonTime.endsWith( + "Z"): adminLogonTime.replace("Z", + "") else: adminLogonTime[0 ..< adminLogonTime.len - + 7] # Remove the timezone + let parsed_4624_logonTime = parse(padString(logonTime, '0', + timeFormat), timeFormat) + let parsed_4672_logonTime = parse(padString(adminLogonTime, '0', + timeFormat), timeFormat) let duration = parsed_4624_logonTime - parsed_4672_logonTime # If the 4624 logon event and 4672 admin logon event are within 10 seconds then flag as an Admin Logon if duration.inSeconds < 10: tableOfResults[]["AdminLogon"] = "Yes" let results = "" & - padString("EID 4624 (Successful Logon): " & intToStr(self.EID_4624_count).insertSep(','), ' ', 80) & - padString("EID 4625 (Failed Logon): " & intToStr(self.EID_4625_count).insertSep(','), ' ', 80) & - padString("EID 4634 (Logoff): " & intToStr(self.EID_4634_count).insertSep(','), ' ', 80) & - padString("EID 4647 (User Initiated Logoff): " & intToStr(self.EID_4647_count).insertSep(','), ' ', 80) & - padString("EID 4648 (Explicit Logon): " & intToStr(self.EID_4648_count).insertSep(','), ' ', 80) & - padString("EID 4672 (Admin Logon): " & intToStr(self.EID_4672_count).insertSep(','), ' ', 80) + padString("EID 4624 (Successful Logon): " & intToStr( + self.EID_4624_count).insertSep(','), ' ', 80) & + padString("EID 4625 (Failed Logon): " & intToStr( + self.EID_4625_count).insertSep(','), ' ', 80) & + padString("EID 4634 (Logoff): " & intToStr( + self.EID_4634_count).insertSep(','), ' ', 80) & + padString("EID 4647 (User Initiated Logoff): " & intToStr( + self.EID_4647_count).insertSep(','), ' ', 80) & + padString("EID 4648 (Explicit Logon): " & intToStr( + self.EID_4648_count).insertSep(','), ' ', 80) & + padString("EID 4672 (Admin Logon): " & intToStr( + self.EID_4672_count).insertSep(','), ' ', 80) if self.displayTable: echo "" echo "Found logon events:" - echo "EID 4624 (Successful Logon): ", intToStr(self.EID_4624_count).insertSep(',') - echo "EID 4625 (Failed Logon): ", intToStr(self.EID_4625_count).insertSep(',') + echo "EID 4624 (Successful Logon): ", intToStr( + self.EID_4624_count).insertSep(',') + echo "EID 4625 (Failed Logon): ", intToStr( + self.EID_4625_count).insertSep(',') echo "EID 4634 (Logoff): ", intToStr(self.EID_4634_count).insertSep(',') - echo "EID 4647 (User Initiated Logoff): ", intToStr(self.EID_4647_count).insertSep(',') - echo "EID 4648 (Explicit Logon): ", intToStr(self.EID_4648_count).insertSep(',') - echo "EID 4672 (Admin Logon): ", intToStr(self.EID_4672_count).insertSep(',') + echo "EID 4647 (User Initiated Logoff): ", intToStr( + self.EID_4647_count).insertSep(',') + echo "EID 4648 (Explicit Logon): ", intToStr( + self.EID_4648_count).insertSep(',') + echo "EID 4672 (Admin Logon): ", intToStr( + self.EID_4672_count).insertSep(',') echo "" # Save results var outputFile = open(self.output, fmWrite) - let header = ["Timestamp", "Channel", "EventID", "Event", "LogoffTime", "ElapsedTime", "FailureReason", "TargetComputer", "TargetUser", "AdminLogon", "SourceComputer", "SourceUser", "SourceIP", "Type", "Impersonation", "ElevatedToken", "Auth", "Process", "LID", "LGUID", "TargetUserSID", "TargetDomainName", "TargetLinkedLID"] + let header = ["Timestamp", "Channel", "EventID", "Event", "LogoffTime", + "ElapsedTime", "FailureReason", "TargetComputer", "TargetUser", + "AdminLogon", "SourceComputer", "SourceUser", "SourceIP", "Type", + "Impersonation", "ElevatedToken", "Auth", "Process", "LID", "LGUID", + "TargetUserSID", "TargetDomainName", "TargetLinkedLID"] ## Write CSV header for h in header: @@ -248,17 +284,21 @@ method resultOutput*(self: TimelineLogonCmd) = let savedFiles = self.output & " (" & formatFileSize(fileSize) & ")" if self.displayTable: echo "Saved results to " & savedFiles - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc timelineLogon(calculateElapsedTime: bool = true, output: string, outputLogoffEvents: bool = false, outputAdminLogonEvents: bool = false, skipProgressBar:bool = false, quiet: bool = false, timeline: string) = +proc timelineLogon(calculateElapsedTime: bool = true, output: string, + outputLogoffEvents: bool = false, outputAdminLogonEvents: bool = false, + skipProgressBar: bool = false, quiet: bool = false, timeline: string) = checkArgs(quiet, timeline, "informational") - let cmd = TimelineLogonCmd( - skipProgressBar: skipProgressBar, - timeline: timeline, - output: output, - name:"timeline-logon", - msg: TimelineLogonMsg, - calculateElapsedTime:calculateElapsedTime, - outputLogoffEvents: outputLogoffEvents, - outputAdminLogonEvents: outputAdminLogonEvents) - cmd.analyzeJSONLFile() \ No newline at end of file + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: + let cmd = TimelineLogonCmd( + skipProgressBar: skipProgressBar, + timeline: timelinePath, + output: output, + name: "timeline-logon", + msg: TimelineLogonMsg, + calculateElapsedTime: calculateElapsedTime, + outputLogoffEvents: outputLogoffEvents, + outputAdminLogonEvents: outputAdminLogonEvents) + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/timelinePartitionDiagnostic.nim b/src/takajopkg/timelinePartitionDiagnostic.nim index ee4d894f..75fed956 100644 --- a/src/takajopkg/timelinePartitionDiagnostic.nim +++ b/src/takajopkg/timelinePartitionDiagnostic.nim @@ -2,103 +2,110 @@ const TimelinePartitionDiagnosticMsg = "This command will create a CSV timeline type TimelinePartitionDiagnosticCmd* = ref object of AbstractCmd - seqOfResultsTables*: seq[Table[string, string]] + seqOfResultsTables*: seq[Table[string, string]] -proc changeByteOrder(vsnStr:string): string = - var res: seq[string] - for i, ch in vsnStr: - if i mod 2 != 0: - res.add(vsnStr[i - 1] & vsnStr[i]) - return join(res.reversed, "") +proc changeByteOrder(vsnStr: string): string = + var res: seq[string] + for i, ch in vsnStr: + if i mod 2 != 0: + res.add(vsnStr[i - 1] & vsnStr[i]) + return join(res.reversed, "") -proc extractVSN(jsonLine: HayabusaJson) : array[4, string] = - var res:array[4, string] = ["n/a", "n/a", "n/a", "n/a"] - if "PartitionStyle" notin jsonLine.ExtraFieldInfo or jsonLine.ExtraFieldInfo["PartitionStyle"].getInt() != 0: - return res - for i, vbrNo in ["Vbr0", "Vbr1", "Vbr2", "Vbr3"]: - if vbrNo notin jsonLine.ExtraFieldInfo: - continue - let vbr = jsonLine.ExtraFieldInfo[vbrNo].getStr() - if vbr.len < 18: - res[i] = "" - continue - let sig = vbr[6..17] - var vsnLittleEndian = "" - if sig == "4D53444F5335" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 141: # FAT32 - vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[134..141] - elif sig == "455846415420" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 207: #ExFAT - vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[200..207] - elif sig == "4E5446532020" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 151: # NTFS - vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[144..151] - if vsnLittleEndian != "": - res[i] = changeByteOrder(vsnLittleEndian) +proc extractVSN(jsonLine: HayabusaJson): array[4, string] = + var res: array[4, string] = ["n/a", "n/a", "n/a", "n/a"] + if "PartitionStyle" notin jsonLine.ExtraFieldInfo or jsonLine.ExtraFieldInfo[ + "PartitionStyle"].getInt() != 0: return res + for i, vbrNo in ["Vbr0", "Vbr1", "Vbr2", "Vbr3"]: + if vbrNo notin jsonLine.ExtraFieldInfo: + continue + let vbr = jsonLine.ExtraFieldInfo[vbrNo].getStr() + if vbr.len < 18: + res[i] = "" + continue + let sig = vbr[6..17] + var vsnLittleEndian = "" + if sig == "4D53444F5335" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 141: # FAT32 + vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[134..141] + elif sig == "455846415420" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 207: #ExFAT + vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[200..207] + elif sig == "4E5446532020" and jsonLine.ExtraFieldInfo[vbrNo].getStr().len > 151: # NTFS + vsnLittleEndian = jsonLine.ExtraFieldInfo[vbrNo].getStr()[144..151] + if vsnLittleEndian != "": + res[i] = changeByteOrder(vsnLittleEndian) + return res -method filter*(self: TimelinePartitionDiagnosticCmd, x: HayabusaJson):bool = - return x.EventId == 1006 and x.Channel == "MS-Win-Partition/Diagnostic" +method filter*(self: TimelinePartitionDiagnosticCmd, x: HayabusaJson): bool = + return x.EventId == 1006 and x.Channel == "MS-Win-Partition/Diagnostic" method analyze*(self: TimelinePartitionDiagnosticCmd, x: HayabusaJson) = - var singleResultTable = initTable[string, string]() - singleResultTable["Timestamp"] = x.Timestamp - singleResultTable["Computer"] = x.Computer - singleResultTable["Manufacturer"] = x.Details["Manufacturer"].getStr() - singleResultTable["Model"] = x.Details["Model"].getStr() - singleResultTable["Revision"] = x.Details["Revision"].getStr() - singleResultTable["SerialNumber"] = x.Details["SerialNumber"].getStr() - for i, vsn in extractVSN(x): - var val = "n/a" - if vsn != "": - val = vsn - singleResultTable["VSN" & intToStr(i)] = val - self.seqOfResultsTables.add(singleResultTable) + var singleResultTable = initTable[string, string]() + singleResultTable["Timestamp"] = x.Timestamp + singleResultTable["Computer"] = x.Computer + singleResultTable["Manufacturer"] = x.Details["Manufacturer"].getStr() + singleResultTable["Model"] = x.Details["Model"].getStr() + singleResultTable["Revision"] = x.Details["Revision"].getStr() + singleResultTable["SerialNumber"] = x.Details["SerialNumber"].getStr() + for i, vsn in extractVSN(x): + var val = "n/a" + if vsn != "": + val = vsn + singleResultTable["VSN" & intToStr(i)] = val + self.seqOfResultsTables.add(singleResultTable) method resultOutput*(self: TimelinePartitionDiagnosticCmd) = - var savedFiles = "n/a" - var results = "n/a" - let header = ["Timestamp", "Computer", "Manufacturer", "Model", "Revision", "SerialNumber", "VSN0", "VSN1", "VSN2", "VSN3"] - if self.output != "": - # Open file to save results - var outputFile = open(self.output, fmWrite) + var savedFiles = "n/a" + var results = "n/a" + let header = ["Timestamp", "Computer", "Manufacturer", "Model", "Revision", + "SerialNumber", "VSN0", "VSN1", "VSN2", "VSN3"] + if self.output != "": + # Open file to save results + var outputFile = open(self.output, fmWrite) - ## Write CSV header - outputFile.write(header.join(",") & "\p") + ## Write CSV header + outputFile.write(header.join(",") & "\p") - ## Write contents - for table in self.seqOfResultsTables: - for i, key in enumerate(header): - if table.hasKey(key): - if i < header.len() - 1: - outputFile.write(escapeCsvField(table[key]) & ",") - else: - outputFile.write(escapeCsvField(table[key])) - else: - outputFile.write(",") - outputFile.write("\p") - outputFile.close() - let fileSize = getFileSize(self.output) - savedFiles = self.output & " (" & formatFileSize(fileSize) & ")" - results = "Events: " & intToStr(self.seqOfResultsTables.len).insertSep(',') - if self.displayTable: - echo "" - echo "Saved results to " & savedFiles - else: - var table: TerminalTable - table.add header - for t in self.seqOfResultsTables: - table.add t[header[0]], t[header[1]], t[header[2]], t[header[3]], t[header[4]], t[header[5]], t[header[6]], t[header[7]], t[header[8]], t[header[9]] - if self.displayTable: - echo "" - table.echoTableSepsWithStyled(seps = boxSeps) - echo "" - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) + ## Write contents + for table in self.seqOfResultsTables: + for i, key in enumerate(header): + if table.hasKey(key): + if i < header.len() - 1: + outputFile.write(escapeCsvField(table[key]) & ",") + else: + outputFile.write(escapeCsvField(table[key])) + else: + outputFile.write(",") + outputFile.write("\p") + outputFile.close() + let fileSize = getFileSize(self.output) + savedFiles = self.output & " (" & formatFileSize(fileSize) & ")" + results = "Events: " & intToStr(self.seqOfResultsTables.len).insertSep(',') + if self.displayTable: + echo "" + echo "Saved results to " & savedFiles + else: + var table: TerminalTable + table.add header + for t in self.seqOfResultsTables: + table.add t[header[0]], t[header[1]], t[header[2]], t[header[3]], t[ + header[4]], t[header[5]], t[header[6]], t[header[7]], t[header[8]], t[ + header[9]] + if self.displayTable: + echo "" + table.echoTableSepsWithStyled(seps = boxSeps) + echo "" + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc timelinePartitionDiagnostic(skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, "informational") +proc timelinePartitionDiagnostic(skipProgressBar: bool = false, + output: string = "", quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, "informational") + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = TimelinePartitionDiagnosticCmd( skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, - name:"timeline-partition-diagnostic", + name: "timeline-partition-diagnostic", msg: TimelinePartitionDiagnosticMsg) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/timelineSuspiciousProcesses.nim b/src/takajopkg/timelineSuspiciousProcesses.nim index 6443bdff..387835cc 100644 --- a/src/takajopkg/timelineSuspiciousProcesses.nim +++ b/src/takajopkg/timelineSuspiciousProcesses.nim @@ -6,218 +6,231 @@ You can change the minimum level with -l, --level=.""" type TimelineSuspiciousProcessesCmd* = ref object of AbstractCmd - level*:string - seqOfResultsTables*: seq[Table[string, string]] - suspicousProcessCount_Sec_4688*, suspicousProcessCount_Sysmon_1* = 0 + level*: string + seqOfResultsTables*: seq[Table[string, string]] + suspicousProcessCount_Sec_4688*, suspicousProcessCount_Sysmon_1 * = 0 -method filter*(self: TimelineSuspiciousProcessesCmd, x: HayabusaJson):bool = - return isMinLevel(x.Level, self.level) and (x.Channel == "Sec" and x.EventId == 4688) or (x.Channel == "Sysmon" and x.EventId == 1) +method filter*(self: TimelineSuspiciousProcessesCmd, x: HayabusaJson): bool = + return isMinLevel(x.Level, self.level) and (x.Channel == "Sec" and + x.EventId == 4688) or (x.Channel == "Sysmon" and x.EventId == 1) method analyze*(self: TimelineSuspiciousProcessesCmd, x: HayabusaJson) = - # Found a Security 4688 process creation event - var - eventId,pidInt = 0 - channel, cmdLine, company, computer, description, eventLevel, eventType, - hashes, hash_MD5, hash_SHA1, hash_SHA256, hash_IMPHASH, - lid, lguid, ruleAuthor,ruleTitle, parentCmdline, - parentGuid, parentPid, pidStr, process, processGuid, product, timestamp, user = "" - let jsonLine = x - if x.Channel == "Sec" and x.EventId == 4688 and isMinLevel(x.Level, self.level): - inc self.suspicousProcessCount_Sec_4688 - try: - cmdLine = jsonLine.Details["Cmdline"].getStr() - except KeyError: - cmdLine = "" - eventType = "Security 4688" - timestamp = jsonLine.Timestamp - ruleTitle = jsonLine.RuleTitle - computer = jsonLine.Computer - process = jsonLine.Details["Proc"].getStr() - pidStr = $jsonLine.Details["PID"].getInt() - user = jsonLine.Details["User"].getStr() - lid = jsonLine.Details["LID"].getStr() - try: - if pidStr == "0": - # -F, --no-field-data-mapping JSONL requires hex conversion - pidStr = intToStr(fromHex[int](jsonLine.Details["PID"].getStr())) - except ValueError: - discard - try: - ruleAuthor = jsonLine.RuleAuthor - except KeyError: - ruleAuthor = "" + # Found a Security 4688 process creation event + var + eventId, pidInt = 0 + channel, cmdLine, company, computer, description, eventLevel, eventType, + hashes, hash_MD5, hash_SHA1, hash_SHA256, hash_IMPHASH, + lid, lguid, ruleAuthor, ruleTitle, parentCmdline, + parentGuid, parentPid, pidStr, process, processGuid, product, timestamp, user = "" + let jsonLine = x + if x.Channel == "Sec" and x.EventId == 4688 and isMinLevel(x.Level, self.level): + inc self.suspicousProcessCount_Sec_4688 + try: + cmdLine = jsonLine.Details["Cmdline"].getStr() + except KeyError: + cmdLine = "" + eventType = "Security 4688" + timestamp = jsonLine.Timestamp + ruleTitle = jsonLine.RuleTitle + computer = jsonLine.Computer + process = jsonLine.Details["Proc"].getStr() + pidStr = $jsonLine.Details["PID"].getInt() + user = jsonLine.Details["User"].getStr() + lid = jsonLine.Details["LID"].getStr() + try: + if pidStr == "0": + # -F, --no-field-data-mapping JSONL requires hex conversion + pidStr = intToStr(fromHex[int](jsonLine.Details["PID"].getStr())) + except ValueError: + discard + try: + ruleAuthor = jsonLine.RuleAuthor + except KeyError: + ruleAuthor = "" - if self.output == "": # Output to screen - echo "Timestamp: " & timestamp - echo "Computer: " & computer - echo "Type: " & eventType - echo "Level: " & eventLevel - echo "Rule: " & ruleTitle - echo "RuleAuthor: " & ruleAuthor - echo "Cmdline: " & cmdLine - echo "Process: " & process - echo "PID: " & pidStr - echo "User: " & user - echo "LID: " & lid - echo "" - else: # Add records to seqOfResultsTables to eventually save to file. - var singleResultTable = initTable[string, string]() - singleResultTable["Timestamp"] = timestamp - singleResultTable["Computer"] = computer - singleResultTable["Type"] = eventType - singleResultTable["Level"] = eventLevel - singleResultTable["Rule"] = ruleTitle - singleResultTable["RuleAuthor"] = ruleAuthor - singleResultTable["Cmdline"] = cmdLine - singleResultTable["Process"] = process - singleResultTable["PID"] = pidStr - singleResultTable["User"] = user - singleResultTable["LID"] = lid - self.seqOfResultsTables.add(singleResultTable) + if self.output == "": # Output to screen + echo "Timestamp: " & timestamp + echo "Computer: " & computer + echo "Type: " & eventType + echo "Level: " & eventLevel + echo "Rule: " & ruleTitle + echo "RuleAuthor: " & ruleAuthor + echo "Cmdline: " & cmdLine + echo "Process: " & process + echo "PID: " & pidStr + echo "User: " & user + echo "LID: " & lid + echo "" + else: # Add records to seqOfResultsTables to eventually save to file. + var singleResultTable = initTable[string, string]() + singleResultTable["Timestamp"] = timestamp + singleResultTable["Computer"] = computer + singleResultTable["Type"] = eventType + singleResultTable["Level"] = eventLevel + singleResultTable["Rule"] = ruleTitle + singleResultTable["RuleAuthor"] = ruleAuthor + singleResultTable["Cmdline"] = cmdLine + singleResultTable["Process"] = process + singleResultTable["PID"] = pidStr + singleResultTable["User"] = user + singleResultTable["LID"] = lid + self.seqOfResultsTables.add(singleResultTable) - # Found a Sysmon 1 process creation event - if x.Channel == "Sysmon" and x.EventId == 1 and isMinLevel(x.Level, self.level): - inc self.suspicousProcessCount_Sysmon_1 - cmdLine = jsonLine.Details["Cmdline"].getStr() - eventType = "Sysmon 1" - timestamp = jsonLine.Timestamp - ruleTitle = jsonLine.RuleTitle - computer = jsonLine.Computer - process = jsonLine.Details["Proc"].getStr() - pidStr = $jsonLine.Details["PID"].getInt() - user = jsonLine.Details["User"].getStr() - lid = jsonLine.Details["LID"].getStr() - lguid = jsonLine.Details["LGUID"].getStr() - processGuid = jsonLine.Details["PGUID"].getStr() - parentCmdline = jsonLine.Details["ParentCmdline"].getStr() - parentPid = $jsonLine.Details["ParentPID"].getInt() - parentGuid = jsonLine.Details["ParentPGUID"].getStr() - description = jsonLine.Details["Description"].getStr() - product = jsonLine.Details["Product"].getStr() - try: - company = jsonLine.Details["Company"].getStr() - except KeyError: - company = "" - try: - ruleAuthor = jsonLine.RuleAuthor - except KeyError: - ruleAuthor = "" - try: - hashes = jsonLine.Details["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist. - let pairs = hashes.split(",") # Split the string into key-value pairs. Ex: MD5=DE9C75F34F47B60A71BBA03760F0579E,SHA256=12F06D3B1601004DB3F7F1A07E7D3AF4CC838E890E0FF50C51E4A0C9366719ED,IMPHASH=336674CB3C8337BDE2C22255345BFF43 - for pair in pairs: - let keyVal = pair.split("=") - case keyVal[0]: - of "MD5": - hash_MD5 = keyVal[1] - of "SHA1": - hash_SHA1 = keyVal[1] - of "SHA256": - hash_SHA256 = keyVal[1] - of "IMPHASH": - hash_IMPHASH = keyVal[1] - except KeyError: - hashes = "" - hash_MD5 = "" - hash_SHA1 = "" - hash_SHA256 = "" - hash_IMPHASH = "" + # Found a Sysmon 1 process creation event + if x.Channel == "Sysmon" and x.EventId == 1 and isMinLevel(x.Level, self.level): + inc self.suspicousProcessCount_Sysmon_1 + cmdLine = jsonLine.Details["Cmdline"].getStr() + eventType = "Sysmon 1" + timestamp = jsonLine.Timestamp + ruleTitle = jsonLine.RuleTitle + computer = jsonLine.Computer + process = jsonLine.Details["Proc"].getStr() + pidStr = $jsonLine.Details["PID"].getInt() + user = jsonLine.Details["User"].getStr() + lid = jsonLine.Details["LID"].getStr() + lguid = jsonLine.Details["LGUID"].getStr() + processGuid = jsonLine.Details["PGUID"].getStr() + parentCmdline = jsonLine.Details["ParentCmdline"].getStr() + parentPid = $jsonLine.Details["ParentPID"].getInt() + parentGuid = jsonLine.Details["ParentPGUID"].getStr() + description = jsonLine.Details["Description"].getStr() + product = jsonLine.Details["Product"].getStr() + try: + company = jsonLine.Details["Company"].getStr() + except KeyError: + company = "" + try: + ruleAuthor = jsonLine.RuleAuthor + except KeyError: + ruleAuthor = "" + try: + hashes = jsonLine.Details["Hashes"].getStr() # Hashes are not enabled by default so this field may not exist. + let pairs = hashes.split(",") # Split the string into key-value pairs. Ex: MD5=DE9C75F34F47B60A71BBA03760F0579E,SHA256=12F06D3B1601004DB3F7F1A07E7D3AF4CC838E890E0FF50C51E4A0C9366719ED,IMPHASH=336674CB3C8337BDE2C22255345BFF43 + for pair in pairs: + let keyVal = pair.split("=") + case keyVal[0]: + of "MD5": + hash_MD5 = keyVal[1] + of "SHA1": + hash_SHA1 = keyVal[1] + of "SHA256": + hash_SHA256 = keyVal[1] + of "IMPHASH": + hash_IMPHASH = keyVal[1] + except KeyError: + hashes = "" + hash_MD5 = "" + hash_SHA1 = "" + hash_SHA256 = "" + hash_IMPHASH = "" - if self.output == "": # Output to screen - echo "Timestamp: " & timestamp - echo "Computer: " & computer - echo "Type: " & eventType - echo "Level: " & eventLevel - echo "Rule: " & ruleTitle - echo "RuleAuthor: " & ruleAuthor - echo "Cmdline: " & cmdLine - echo "Process: " & process - echo "PID: " & pidStr - echo "User: " & user - echo "LID: " & lid - echo "LGUID: " & lguid - echo "ProcessGUID: " & processGuid - echo "ParentCmdline: " & parentCmdline - echo "ParentPID: " & parentPid - echo "ParentGUID: " & parentGuid - echo "Description: " & description - echo "Product: " & product - echo "Company: " & company - echo "MD5 Hash: " & hash_MD5 - echo "SHA1 Hash: " & hash_SHA1 - echo "SHA256 Hash: " & hash_SHA256 - echo "Import Hash: " & hash_IMPHASH - echo "" - else: # Add records to seqOfResultsTables to eventually save to file. - var singleResultTable = initTable[string, string]() - singleResultTable["Timestamp"] = timestamp - singleResultTable["Computer"] = computer - singleResultTable["Type"] = eventType - singleResultTable["Level"] = eventLevel - singleResultTable["Rule"] = ruleTitle - singleResultTable["RuleAuthor"] = ruleAuthor - singleResultTable["Cmdline"] = cmdLine - singleResultTable["Process"] = process - singleResultTable["PID"] = pidStr - singleResultTable["User"] = user - singleResultTable["LID"] = lid - singleResultTable["LGUID"] = lguid - singleResultTable["ProcessGUID"] = processGuid - singleResultTable["ParentCmdline"] = parentCmdline - singleResultTable["ParentPID"] = parentPid - singleResultTable["ParentPGUID"] = parentGuid - singleResultTable["Description"] = description - singleResultTable["Product"] = product - singleResultTable["Company"] = company - singleResultTable["MD5 Hash"] = hash_MD5 - singleResultTable["SHA1 Hash"] = hash_SHA1 - singleResultTable["SHA256 Hash"] = hash_SHA256 - singleResultTable["Import Hash"] = hash_IMPHASH - self.seqOfResultsTables.add(singleResultTable) + if self.output == "": # Output to screen + echo "Timestamp: " & timestamp + echo "Computer: " & computer + echo "Type: " & eventType + echo "Level: " & eventLevel + echo "Rule: " & ruleTitle + echo "RuleAuthor: " & ruleAuthor + echo "Cmdline: " & cmdLine + echo "Process: " & process + echo "PID: " & pidStr + echo "User: " & user + echo "LID: " & lid + echo "LGUID: " & lguid + echo "ProcessGUID: " & processGuid + echo "ParentCmdline: " & parentCmdline + echo "ParentPID: " & parentPid + echo "ParentGUID: " & parentGuid + echo "Description: " & description + echo "Product: " & product + echo "Company: " & company + echo "MD5 Hash: " & hash_MD5 + echo "SHA1 Hash: " & hash_SHA1 + echo "SHA256 Hash: " & hash_SHA256 + echo "Import Hash: " & hash_IMPHASH + echo "" + else: # Add records to seqOfResultsTables to eventually save to file. + var singleResultTable = initTable[string, string]() + singleResultTable["Timestamp"] = timestamp + singleResultTable["Computer"] = computer + singleResultTable["Type"] = eventType + singleResultTable["Level"] = eventLevel + singleResultTable["Rule"] = ruleTitle + singleResultTable["RuleAuthor"] = ruleAuthor + singleResultTable["Cmdline"] = cmdLine + singleResultTable["Process"] = process + singleResultTable["PID"] = pidStr + singleResultTable["User"] = user + singleResultTable["LID"] = lid + singleResultTable["LGUID"] = lguid + singleResultTable["ProcessGUID"] = processGuid + singleResultTable["ParentCmdline"] = parentCmdline + singleResultTable["ParentPID"] = parentPid + singleResultTable["ParentPGUID"] = parentGuid + singleResultTable["Description"] = description + singleResultTable["Product"] = product + singleResultTable["Company"] = company + singleResultTable["MD5 Hash"] = hash_MD5 + singleResultTable["SHA1 Hash"] = hash_SHA1 + singleResultTable["SHA256 Hash"] = hash_SHA256 + singleResultTable["Import Hash"] = hash_IMPHASH + self.seqOfResultsTables.add(singleResultTable) method resultOutput*(self: TimelineSuspiciousProcessesCmd) = - var savedFiles = "n/a" - var results = "n/a" - if self.output != "" and (self.suspicousProcessCount_Sec_4688 > 0 or self.suspicousProcessCount_Sysmon_1 > 0): # Save results to CSV + var savedFiles = "n/a" + var results = "n/a" + if self.output != "" and (self.suspicousProcessCount_Sec_4688 > 0 or + self.suspicousProcessCount_Sysmon_1 > 0): # Save results to CSV # Open file to save results - let header = ["Timestamp", "Computer", "Type", "Level", "Rule", "RuleAuthor", "Cmdline", "Process", "PID", "User", "LID", "LGUID", "ProcessGUID", "ParentCmdline", "ParentPID", "ParentPGUID", "Description", "Product", "Company", "MD5 Hash", "SHA1 Hash", "SHA256 Hash", "Import Hash"] - var outputFile = open(self.output, fmWrite) + let header = ["Timestamp", "Computer", "Type", "Level", "Rule", + "RuleAuthor", "Cmdline", "Process", "PID", "User", "LID", "LGUID", + "ProcessGUID", "ParentCmdline", "ParentPID", "ParentPGUID", + "Description", "Product", "Company", "MD5 Hash", "SHA1 Hash", + "SHA256 Hash", "Import Hash"] + var outputFile = open(self.output, fmWrite) - ## Write CSV header - outputFile.write(header.join(",") & "\p") + ## Write CSV header + outputFile.write(header.join(",") & "\p") - ## Write contents - for table in self.seqOfResultsTables: - for key in header: - if table.hasKey(key): - outputFile.write(escapeCsvField(table[key]) & ",") - else: - outputFile.write(",") - outputFile.write("\p") - outputFile.close() - let fileSize = getFileSize(self.output) - savedFiles = self.output & " (" & formatFileSize(fileSize) & ")" - results = "Events: " & intToStr(self.seqOfResultsTables.len).insertSep(',') - if self.displayTable: - echo "" - echo "Saved results to " & savedFiles - if self.displayTable: - if self.suspicousProcessCount_Sec_4688 == 0 and self.suspicousProcessCount_Sysmon_1 == 0: - echo "" - echo "No suspicous processes were found. There are either no malicious processes or you need to change the level." + ## Write contents + for table in self.seqOfResultsTables: + for key in header: + if table.hasKey(key): + outputFile.write(escapeCsvField(table[key]) & ",") else: - echo "" - echo "Suspicious processes in Security 4688 process creation events: " & intToStr(self.suspicousProcessCount_Sec_4688).insertSep(',') - echo "Suspicious processes in Sysmon 1 process creation events: " & intToStr(self.suspicousProcessCount_Sysmon_1).insertSep(',') - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) + outputFile.write(",") + outputFile.write("\p") + outputFile.close() + let fileSize = getFileSize(self.output) + savedFiles = self.output & " (" & formatFileSize(fileSize) & ")" + results = "Events: " & intToStr(self.seqOfResultsTables.len).insertSep(',') + if self.displayTable: + echo "" + echo "Saved results to " & savedFiles + if self.displayTable: + if self.suspicousProcessCount_Sec_4688 == 0 and + self.suspicousProcessCount_Sysmon_1 == 0: + echo "" + echo "No suspicous processes were found. There are either no malicious processes or you need to change the level." + else: + echo "" + echo "Suspicious processes in Security 4688 process creation events: " & + intToStr(self.suspicousProcessCount_Sec_4688).insertSep(',') + echo "Suspicious processes in Sysmon 1 process creation events: " & + intToStr(self.suspicousProcessCount_Sysmon_1).insertSep(',') + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc timelineSuspiciousProcesses(level: string = "high", skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, "informational") +proc timelineSuspiciousProcesses(level: string = "high", + skipProgressBar: bool = false, output: string = "", quiet: bool = false, + timeline: string) = + checkArgs(quiet, timeline, "informational") + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = TimelineSuspiciousProcessesCmd( skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, level: level, - name:"timeline-suspicious-processes", + name: "timeline-suspicious-processes", msg: TimelineSuspiciousProcessesMsg) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/timelineTasks.nim b/src/takajopkg/timelineTasks.nim index f513217c..00f930c3 100644 --- a/src/takajopkg/timelineTasks.nim +++ b/src/takajopkg/timelineTasks.nim @@ -2,110 +2,116 @@ const TimelineTasksMsg = "This command creates a CSV timeline of scheduled task type TimelineTasksCmd* = ref object of AbstractCmd - rowData* = newSeq[(OrderedTable[string, string], OrderedTable[string, string])]() - outputLogoffEvents: bool + rowData* = newSeq[(OrderedTable[string, string], OrderedTable[string, + string])]() + outputLogoffEvents: bool -proc extractText(allXmlNodes:XmlNode, tag:string): string = - let xmlNodes = allXmlNodes.findAll(tag) - if len(xmlNodes) > 0: - var xml = $xmlNodes[0] - return decodeEntity(xml.replace("<" & tag & ">", "").replace("","")) - return "" +proc extractText(allXmlNodes: XmlNode, tag: string): string = + let xmlNodes = allXmlNodes.findAll(tag) + if len(xmlNodes) > 0: + var xml = $xmlNodes[0] + return decodeEntity(xml.replace("<" & tag & ">", "").replace("", "")) + return "" -proc walkXml(node:XmlNode, tagName: string, resuls: var OrderedTable[string, string]) = +proc walkXml(node: XmlNode, tagName: string, resuls: var OrderedTable[string, string]) = if node.len == 0 and node.innerText.len != 0: - resuls[tagName] = node.innerText - return + resuls[tagName] = node.innerText + return for child in node: - if tagName != tag(node): - walkXml(child, tagName & ":" & tag(node), resuls) - else: - walkXml(child, tag(node), resuls) + if tagName != tag(node): + walkXml(child, tagName & ":" & tag(node), resuls) + else: + walkXml(child, tag(node), resuls) -method filter*(self: TimelineTasksCmd, x: HayabusaJson):bool = - return x.EventId == 4698 and x.Channel == "Sec" +method filter*(self: TimelineTasksCmd, x: HayabusaJson): bool = + return x.EventId == 4698 and x.Channel == "Sec" method analyze*(self: TimelineTasksCmd, x: HayabusaJson) = - let ts = x.Timestamp - let taskName = x.Details["Name"].getStr("N/A") - let subjectUserName = x.Details["User"].getStr("N/A") - let subjectUserSid = x.ExtraFieldInfo["SubjectUserSid"].getStr("N/A") - let subjectDomainName = x.ExtraFieldInfo["SubjectDomainName"].getStr("N/A") - let subjectLogonId = x.Details["LID"].getStr("N/A") - let content = x.Details["Content"].getStr("N/A").replace("\\r\\n", "") - var basicTable = initOrderedTable[string, string]() - basicTable["Timestamp"] = ts - basicTable["TaskName"] = taskName - basicTable["User"] = subjectUserName - basicTable["SubjectUserSid"] = subjectUserSid - basicTable["SubjectDomainName"] = subjectDomainName - basicTable["SubjectLogonId"] = subjectLogonId - var detailedTable = initOrderedTable[string, string]() - try: - let rootNode = parseXml(content) - basicTable["Command"] = extractText(rootNode, "Command") - basicTable["Arguments"] = extractText(rootNode, "Arguments") - for tagName in ["Triggers", "Principal", "Settings", "RegistrationInfo"]: - for i, node in enumerate(rootNode.findAll(tagName)): - if i == 0: - walkXml(node, tagName , detailedTable) - else: - walkXml(node, tagName & "[" & intToStr(i) & "]", detailedTable) - except XmlError: - return - self.rowData.add((basicTable, detailedTable)) + let ts = x.Timestamp + let taskName = x.Details["Name"].getStr("N/A") + let subjectUserName = x.Details["User"].getStr("N/A") + let subjectUserSid = x.ExtraFieldInfo["SubjectUserSid"].getStr("N/A") + let subjectDomainName = x.ExtraFieldInfo["SubjectDomainName"].getStr("N/A") + let subjectLogonId = x.Details["LID"].getStr("N/A") + let content = x.Details["Content"].getStr("N/A").replace("\\r\\n", "") + var basicTable = initOrderedTable[string, string]() + basicTable["Timestamp"] = ts + basicTable["TaskName"] = taskName + basicTable["User"] = subjectUserName + basicTable["SubjectUserSid"] = subjectUserSid + basicTable["SubjectDomainName"] = subjectDomainName + basicTable["SubjectLogonId"] = subjectLogonId + var detailedTable = initOrderedTable[string, string]() + try: + let rootNode = parseXml(content) + basicTable["Command"] = extractText(rootNode, "Command") + basicTable["Arguments"] = extractText(rootNode, "Arguments") + for tagName in ["Triggers", "Principal", "Settings", "RegistrationInfo"]: + for i, node in enumerate(rootNode.findAll(tagName)): + if i == 0: + walkXml(node, tagName, detailedTable) + else: + walkXml(node, tagName & "[" & intToStr(i) & "]", detailedTable) + except XmlError: + return + self.rowData.add((basicTable, detailedTable)) -method resultOutput*(self: TimelineTasksCmd)= - var savedFiles = "n/a" - var results = "n/a" - if self.rowData.len == 0: - if self.displayTable: - echo "" - echo "No scheduled task events were found." - else: - var allDetailedKeys = initOrderedSet[string]() - for (_, detailedTable) in self.rowData: - for key in detailedTable.keys: - allDetailedKeys.incl(key) - let basicHeader = @["Timestamp", "TaskName", "User", "SubjectUserSid", "SubjectDomainName", "SubjectLogonId", "Command", "Arguments"] - let detailedHeader = toSeq(allDetailedKeys).sorted +method resultOutput*(self: TimelineTasksCmd) = + var savedFiles = "n/a" + var results = "n/a" + if self.rowData.len == 0: + if self.displayTable: + echo "" + echo "No scheduled task events were found." + else: + var allDetailedKeys = initOrderedSet[string]() + for (_, detailedTable) in self.rowData: + for key in detailedTable.keys: + allDetailedKeys.incl(key) + let basicHeader = @["Timestamp", "TaskName", "User", "SubjectUserSid", + "SubjectDomainName", "SubjectLogonId", "Command", "Arguments"] + let detailedHeader = toSeq(allDetailedKeys).sorted - # Save results - var outputFile = open(self.output, fmWrite) + # Save results + var outputFile = open(self.output, fmAppend) - ## Write CSV header - let header = concat(basicHeader, detailedHeader) - for h in header: - outputFile.write(h & ",") - outputFile.write("\p") + ## Write CSV header + let header = concat(basicHeader, detailedHeader) + for h in header: + outputFile.write(h & ",") + outputFile.write("\p") - ## Write contents - for (basicTable, detailedTable) in self.rowData: - for i, columnName in enumerate(header): - if columnName in basicHeader: - outputFile.write(basicTable[columnName] & ",") - elif i > basicHeader.len - 1: - if columnName in detailedTable: - outputFile.write(escapeCsvField(detailedTable[columnName]) & ",") - else: - outputFile.write(",") - outputFile.write("\p") - outputFile.close() - let fileSize = getFileSize(self.output) - savedFiles = self.output & " (" & formatFileSize(fileSize) & ")" - results = "Events: " & intToStr(self.rowData.len).insertSep(',') - if self.displayTable: - echo "" - echo "Saved results to " & savedFiles - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) + ## Write contents + for (basicTable, detailedTable) in self.rowData: + for i, columnName in enumerate(header): + if columnName in basicHeader: + outputFile.write(basicTable[columnName] & ",") + elif i > basicHeader.len - 1: + if columnName in detailedTable: + outputFile.write(escapeCsvField(detailedTable[columnName]) & ",") + else: + outputFile.write(",") + outputFile.write("\p") + outputFile.close() + let fileSize = getFileSize(self.output) + savedFiles = self.output & " (" & formatFileSize(fileSize) & ")" + results = "Events: " & intToStr(self.rowData.len).insertSep(',') + if self.displayTable: + echo "" + echo "Saved results to " & savedFiles + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc timelineTasks(skipProgressBar:bool = false, output: string, outputLogoffEvents: bool = false, quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, "informational") +proc timelineTasks(skipProgressBar: bool = false, output: string, + outputLogoffEvents: bool = false, quiet: bool = false, timeline: string) = + checkArgs(quiet, timeline, "informational") + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = TimelineTasksCmd( skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, - name:"timeline-tasks", + name: "timeline-tasks", msg: TimelineTasksMsg, outputLogoffEvents: outputLogoffEvents) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/ttpResult.nim b/src/takajopkg/ttpResult.nim index 6a20cccf..56fc66f9 100644 --- a/src/takajopkg/ttpResult.nim +++ b/src/takajopkg/ttpResult.nim @@ -5,62 +5,65 @@ import std/tables import std/sequtils type TTPResult* = ref object - techniqueID: string - comment: string - score: int + techniqueID: string + comment: string + score: int -proc newTTPResult*(techniqueID: string, comment: string, score: int): TTPResult = - result = TTPResult(techniqueID: techniqueID, comment: comment, score:score) +proc newTTPResult*(techniqueID: string, comment: string, + score: int): TTPResult = + result = TTPResult(techniqueID: techniqueID, comment: comment, score: score) -proc outputTTPResult*(stackedMitreTags:Table[string, string], stackedMitreTagsCount:Table[string, int], output:string, displayTable:bool = true, name:string):string = - var savedFiles = "n/a" - if stackedMitreTags.len == 0: - savedFiles = "No MITRE ATT&CK tags were found in the Hayabusa results." - if displayTable: - echo "" - echo savedFiles - echo "Please run your Hayabusa scan with a profile that includes the %MitreTags% field. (ex: -p verbose)" - else: - var mitreTags = newSeq[TTPResult]() - let maxCount = stackedMitreTagsCount.values.toSeq.max - for techniqueID, ruleTitle in stackedMitreTags: - let score = toInt(round(stackedMitreTagsCount[techniqueID]/maxCount * 100)) - mitreTags.add(newTTPResult(techniqueID, ruleTitle, score)) - let jsonObj = %* { - "name": name, - "versions": { - "attack": "14", - "navigator": "4.9.1", - "layer": "4.5" - }, - "domain": "enterprise-attack", - "description": name, - "techniques": mitreTags, - "gradient": { - "colors": [ - "#8ec843ff", - "#ffe766ff", - "#ff6666ff" - ], - "minValue": 0, - "maxValue": 100 - }, - "legendItems": [], - "metadata": [], - "links": [], - "showTacticRowBackground": false, - "tacticRowBackground": "#dddddd", - "selectTechniquesAcrossTactics": true, - "selectSubtechniquesWithParent": false, - "selectVisibleTechniques": false - } +proc outputTTPResult*(stackedMitreTags: Table[string, string], + stackedMitreTagsCount: Table[string, int], output: string, + displayTable: bool = true, name: string): string = + var savedFiles = "n/a" + if stackedMitreTags.len == 0: + savedFiles = "No MITRE ATT&CK tags were found in the Hayabusa results." + if displayTable: + echo "" + echo savedFiles + echo "Please run your Hayabusa scan with a profile that includes the %MitreTags% field. (ex: -p verbose)" + else: + var mitreTags = newSeq[TTPResult]() + let maxCount = stackedMitreTagsCount.values.toSeq.max + for techniqueID, ruleTitle in stackedMitreTags: + let score = toInt(round(stackedMitreTagsCount[techniqueID]/maxCount * 100)) + mitreTags.add(newTTPResult(techniqueID, ruleTitle, score)) + let jsonObj = %* { + "name": name, + "versions": { + "attack": "14", + "navigator": "4.9.1", + "layer": "4.5" + }, + "domain": "enterprise-attack", + "description": name, + "techniques": mitreTags, + "gradient": { + "colors": [ + "#8ec843ff", + "#ffe766ff", + "#ff6666ff" + ], + "minValue": 0, + "maxValue": 100 + }, + "legendItems": [], + "metadata": [], + "links": [], + "showTacticRowBackground": false, + "tacticRowBackground": "#dddddd", + "selectTechniquesAcrossTactics": true, + "selectSubtechniquesWithParent": false, + "selectVisibleTechniques": false + } - let outputFile = open(output, FileMode.fmWrite) - outputFile.write(jsonObj.pretty()) - let outputFileSize = getFileSize(outputFile) - savedFiles = output & " (" & formatFileSize(outputFileSize) & ")" - outputFile.close() - if displayTable: - echo "" - echo "Saved file: " & savedFiles - return savedFiles \ No newline at end of file + let outputFile = open(output, FileMode.fmAppend) + outputFile.write(jsonObj.pretty()) + let outputFileSize = getFileSize(outputFile) + savedFiles = output & " (" & formatFileSize(outputFileSize) & ")" + outputFile.close() + if displayTable: + echo "" + echo "Saved file: " & savedFiles + return savedFiles diff --git a/src/takajopkg/ttpSummary.nim b/src/takajopkg/ttpSummary.nim index f94b3a0b..5f3db0e0 100644 --- a/src/takajopkg/ttpSummary.nim +++ b/src/takajopkg/ttpSummary.nim @@ -20,24 +20,24 @@ proc compareArrays(a, b: array[5, string]): int = return 0 type - TTPSummaryCmd* = ref object of AbstractCmd - seqOfResultsTables*: seq[array[5, string]] - techniqueIDs*:HashSet[string] = initHashSet[string]() - attack*: JsonNode - tac_no* = {"Reconnaissance": "01. ", - "Resource Development": "02. ", - "Initial Access": "03. ", - "Execution": "04. ", - "Persistence": "05. ", - "Privilege Escalation": "06. ", - "Defense Evasion": "07. ", - "Credential Access": "08. ", - "Discovery": "09. ", - "Lateral Movement": "10. ", - "Collection": "11. ", - "Exfiltration": "12. ", - "Command and Control": "13. ", - "Impact": "14. "}.toTable + TTPSummaryCmd* = ref object of AbstractCmd + seqOfResultsTables*: seq[array[5, string]] + techniqueIDs*: HashSet[string] = initHashSet[string]() + attack*: JsonNode + tac_no* = {"Reconnaissance": "01. ", + "Resource Development": "02. ", + "Initial Access": "03. ", + "Execution": "04. ", + "Persistence": "05. ", + "Privilege Escalation": "06. ", + "Defense Evasion": "07. ", + "Credential Access": "08. ", + "Discovery": "09. ", + "Lateral Movement": "10. ", + "Collection": "11. ", + "Exfiltration": "12. ", + "Command and Control": "13. ", + "Impact": "14. "}.toTable method analyze*(self: TTPSummaryCmd, x: HayabusaJson) = try: @@ -60,13 +60,14 @@ method resultOutput*(self: TTPSummaryCmd) = self.seqOfResultsTables.sort(compareArrays) var savedFiles = "n/a" var results = "n/a" - let header = ["Computer", "Tactic", "Technique", "Sub-Technique", "RuleTitle", "Count"] - var prev = ["","","","",""] + let header = ["Computer", "Tactic", "Technique", "Sub-Technique", + "RuleTitle", "Count"] + var prev = ["", "", "", "", ""] var count = 1 var ruleStr = initHashSet[string]() if self.output != "": # Open file to save results - var outputFile = open(self.output, fmWrite) + var outputFile = open(self.output, fmAppend) ## Write CSV header outputFile.write(header.join(",") & "\p") @@ -101,7 +102,8 @@ method resultOutput*(self: TTPSummaryCmd) = if arr[0..<4] == prev[0..<4]: count += 1 continue - table.add arr[0], arr[1], arr[2], arr[3], ruleStr.mapIt($it).join(", "), intToStr(count) + table.add arr[0], arr[1], arr[2], arr[3], ruleStr.mapIt($it).join( + ", "), intToStr(count) prev = arr count = 1 ruleStr = initHashSet[string]() @@ -112,18 +114,21 @@ method resultOutput*(self: TTPSummaryCmd) = if self.displayTable: echo savedFiles echo "Please run your Hayabusa scan with a profile that includes the %MitreTags% field. (ex: -p verbose)" - self.cmdResult = CmdResult(results:results, savedFiles:savedFiles) + self.cmdResult = CmdResult(results: results, savedFiles: savedFiles) -proc ttpSummary(skipProgressBar:bool = false, output: string = "", quiet: bool = false, timeline: string) = +proc ttpSummary(skipProgressBar: bool = false, output: string = "", + quiet: bool = false, timeline: string) = checkArgs(quiet, timeline, "informational") if not os.fileExists("mitre-attack.json"): echo "The file '" & "mitre-attack.json" & "' does not exist. Please specify a valid file path." quit(1) - let cmd = TTPSummaryCmd( - skipProgressBar: skipProgressBar, - timeline: timeline, - output: output, - name:"ttp-summary", - msg: TTPSummaryMsg) - cmd.attack = readJsonFromFile("mitre-attack.json") - cmd.analyzeJSONLFile() \ No newline at end of file + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: + let cmd = TTPSummaryCmd( + skipProgressBar: skipProgressBar, + timeline: timelinePath, + output: output, + name: "ttp-summary", + msg: TTPSummaryMsg) + cmd.attack = readJsonFromFile("mitre-attack.json") + cmd.analyzeJSONLFile() diff --git a/src/takajopkg/ttpVisualize.nim b/src/takajopkg/ttpVisualize.nim index 8d87d889..4424eb29 100644 --- a/src/takajopkg/ttpVisualize.nim +++ b/src/takajopkg/ttpVisualize.nim @@ -6,29 +6,37 @@ type stackedMitreTagsCount* = initTable[string, int]() method analyze*(self: TTPVisualizeCmd, x: HayabusaJson) = - try: - for tag in x.MitreTags: - let techniqueID = tag - let ruleTitle = strip(x.RuleTitle) - if self.stackedMitreTags.hasKey(techniqueID) and ruleTitle notin self.stackedMitreTags[techniqueID]: - self.stackedMitreTags[techniqueID] = self.stackedMitreTags[techniqueID] & "," & ruleTitle - self.stackedMitreTagsCount[techniqueID] += 1 - else: - self.stackedMitreTags[techniqueID] = ruleTitle - self.stackedMitreTagsCount[techniqueID] = 1 - except CatchableError: - discard + try: + for tag in x.MitreTags: + let techniqueID = tag + let ruleTitle = strip(x.RuleTitle) + if self.stackedMitreTags.hasKey(techniqueID) and ruleTitle notin + self.stackedMitreTags[techniqueID]: + self.stackedMitreTags[techniqueID] = self.stackedMitreTags[ + techniqueID] & "," & ruleTitle + self.stackedMitreTagsCount[techniqueID] += 1 + else: + self.stackedMitreTags[techniqueID] = ruleTitle + self.stackedMitreTagsCount[techniqueID] = 1 + except CatchableError: + discard method resultOutput*(self: TTPVisualizeCmd) = - let savedFiles = outputTTPResult(self.stackedMitreTags, self.stackedMitreTagsCount, self.output, false, "Hayabusa detection result heatmap") - self.cmdResult = CmdResult(results:"You can import this into ATT&CK Navigator", savedFiles:savedFiles) + let savedFiles = outputTTPResult(self.stackedMitreTags, + self.stackedMitreTagsCount, self.output, false, "Hayabusa detection result heatmap") + self.cmdResult = CmdResult(results: "You can import this into ATT&CK Navigator", + savedFiles: savedFiles) -proc ttpVisualize(skipProgressBar:bool = false, output: string = "mitre-ttp-heatmap.json", quiet: bool = false, timeline: string) = - checkArgs(quiet, timeline, "informational") +proc ttpVisualize(skipProgressBar: bool = false, + output: string = "mitre-ttp-heatmap.json", quiet: bool = false, + timeline: string) = + checkArgs(quiet, timeline, "informational") + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: let cmd = TTPVisualizeCmd( skipProgressBar: skipProgressBar, - timeline: timeline, + timeline: timelinePath, output: output, - name:"ttp-visualize", + name: "ttp-visualize", msg: TTPVisualizeMsg) - cmd.analyzeJSONLFile() \ No newline at end of file + cmd.analyzeJSONLFile() From 82cd2c84a132effbd3926bb4556a95c1daa50dff Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Sat, 23 Mar 2024 21:53:45 +0900 Subject: [PATCH 2/9] test(testsubmodule): due to changed getTargetExtFileLists function arguments #133 --- tests/testsubmodule.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/testsubmodule.nim b/tests/testsubmodule.nim index 4cf71d5e..8f0a4e76 100644 --- a/tests/testsubmodule.nim +++ b/tests/testsubmodule.nim @@ -38,4 +38,4 @@ test "csv file path import": test "check getYMLLists": let expect = @["1.yml"] - check getTargetExtFileLists("./tests", ".yml") == expect + check getTargetExtFileLists("./tests", ".yml", false) == expect From fc4bf60436719ca6dd1545d54ecd28a17dc99f7b Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Tue, 26 Mar 2024 13:32:21 +0900 Subject: [PATCH 3/9] fix(general): fixed checkArgs quit condition #133 --- src/takajopkg/general.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/takajopkg/general.nim b/src/takajopkg/general.nim index bf4048af..6db2642f 100644 --- a/src/takajopkg/general.nim +++ b/src/takajopkg/general.nim @@ -374,7 +374,11 @@ proc checkArgs*(quiet: bool = false, timeline: string, level: string) = if not quiet: styledEcho(fgGreen, outputLogo()) - if not os.fileExists(timeline) or os.dirExists(timeline): + let fileInfo = getFileInfo(timeline) + if ((fileInfo.kind == pcDir or fileInfo.kind == pcLinkToDir) and + not os.dirExists(timeline)) or ((fileInfo.kind == + pcFile or fileInfo.kind == pcLinkToFile) and + not os.fileExists(timeline)): echo "The file '" & timeline & "' does not exist. Please specify a valid file path." quit(1) From 1c0296155c9bf64895a16d2934adc6daa56864f2 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Wed, 27 Mar 2024 08:55:21 +0900 Subject: [PATCH 4/9] fix(general): modified to adapt directory path #133 --- src/takajopkg/general.nim | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/takajopkg/general.nim b/src/takajopkg/general.nim index 6db2642f..fc6bed8f 100644 --- a/src/takajopkg/general.nim +++ b/src/takajopkg/general.nim @@ -382,8 +382,10 @@ proc checkArgs*(quiet: bool = false, timeline: string, level: string) = echo "The file '" & timeline & "' does not exist. Please specify a valid file path." quit(1) - if not isJsonConvertible(timeline): - quit(1) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + for timelinePath in filePaths: + if not isJsonConvertible(timelinePath): + quit(1) if level != "critical" and level != "high" and level != "medium" and level != "low" and level != "informational": From b816f9c96f32e47c79fbc6c4c0ba27ad84290398 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 29 Mar 2024 01:53:10 +0900 Subject: [PATCH 5/9] fix: fixed overwrite result file when timeline option is directory #133 --- src/takajopkg/automagic.nim | 262 +++++++++++++------------- src/takajopkg/extractScriptblocks.nim | 6 +- src/takajopkg/general.nim | 36 ++-- src/takajopkg/listDomains.nim | 2 +- src/takajopkg/splitCsvTimeline.nim | 2 +- src/takajopkg/takajoCore.nim | 107 ++++++----- src/takajopkg/timelineTasks.nim | 2 +- src/takajopkg/ttpResult.nim | 2 +- src/takajopkg/ttpSummary.nim | 2 +- 9 files changed, 216 insertions(+), 205 deletions(-) diff --git a/src/takajopkg/automagic.nim b/src/takajopkg/automagic.nim index 03c9a3a1..640e5573 100644 --- a/src/takajopkg/automagic.nim +++ b/src/takajopkg/automagic.nim @@ -14,135 +14,133 @@ proc autoMagic(level: string = "low", skipProgressBar: bool = false, else: createDir(output) - var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) - for timelinePath in filePaths: - # extract-scriptblocks -t ../hayabusa/timeline.jsonl -l -o case-1/scriptblock-logs/ - let cmd1 = ExtractScriptBlocksCmd(name: "extract-scriptblocks", - level: level, skipProgressBar: skipProgressBar, - displayTable: displayTable, - timeline: timelinePath, output: output & "/scriptblock-logs") - if not dirExists(output & "/scriptblock-logs/"): - createDir(output & "/scriptblock-logs/") - - # list-domains -t ../hayabusa/timeline.jsonl -o case-1/ListDomains.txt - let cmd2 = ListDomainsCmd(name: "list-domains", displayTable: displayTable, - timeline: timelinePath, output: output & "/ListDomains.txt") - - # list-domains -t ../hayabusa/timeline.jsonl -d -w -o case-1/ListDomains-Detailed.txt - let cmd3 = ListDomainsCmd(name: "list-domains(detailed)", - displayTable: displayTable, timeline: timelinePath, output: output & - "/ListDomainsDetailed.txt", includeSubdomains: true, - includeWorkstations: true) - - # list-hashes -t ../hayabusa/timeline.jsonl -l -o case-1/hashes - let cmd4 = ListHashesCmd(name: "list-hashes", displayTable: displayTable, - timeline: timelinePath, level: level, output: output & "/ListHashes") - - # list-ip-addresses -t ../hayabusa/timeline.jsonl -o case-1/IP-Addresses.txt - let cmd5 = ListIpAddressesCmd(name: "list-ip-addresses", - displayTable: displayTable, timeline: timelinePath, output: output & "/ListIP-Addresses.txt") - - # stack-cmdlines -t ../hayabusa/timeline.jsonl --level -o case-1/cmdlines.csv - let cmd6 = StackCmdlineCmd(name: "stack-cmdlines", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & "/StackCmdlines.csv") - - # stack-computers -t ../hayabusa/timeline.jsonl --level -o case-1/TargetComputers.csv - let cmd7 = StackComputersCmd(name: "stack-computers", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & "/StackTargetComputers.csv") - - # stack-computers -t ../hayabusa/timeline.jsonl --level --sourceComputers -o case-1/SourceComputers.csv - let cmd8 = StackComputersCmd(name: "stack-computers", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & - "/StackSourceComputers.csv", sourceComputers: true) - - # stack-dns -t ../hayabusa/timeline.jsonl --level -o case-1/DNS.csv - let cmd9 = StackDNSCmd(name: "stack-dns", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & "/StackDNS.csv") - - # stack-ip-addresses -t ../hayabusa/timeline.jsonl --level -o case-1/SourceIP-Addresses.csv - let cmd10 = StackIpAddressesCmd(name: "stack-ip-addresses(source)", - level: level, displayTable: displayTable, timeline: timelinePath, - output: output & "/StackSourceIP-Addresses.csv") - - # stack-ip-addresses -t ../hayabusa/timeline.jsonl --level --targetIpAddresses -o case-1/TargetIP-Addresses.csv - let cmd11 = StackIpAddressesCmd(name: "stack-ip-addresses(target)", - level: level, displayTable: displayTable, timeline: timelinePath, - output: output & "/StackTargetIP-Addresses.csv", - targetIpAddresses: true) - - # stack-logons -t ../hayabusa/timeline.jsonl -o case-1/Logons.csv - let cmd12 = StackLogonsCmd(name: "stack-logons", displayTable: displayTable, - timeline: timelinePath, output: output & "/StackLogons.csv") - - # stack-processes -t ../hayabusa/timeline.jsonl --level -o case-1/Processes.csv - let cmd13 = StackProcessesCmd(name: "stack-processes", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & "/StackProcesses.csv") - - # stack-services -t ../hayabusa/timeline.jsonl --level -o case-1/Services.csv - let cmd14 = StackServicesCmd(name: "stack-services", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & "/StackServices.csv") - - # stack-tasks -t ../hayabusa/timeline.jsonl --level -o case-1/Tasks.csv - let cmd15 = StackTasksCmd(name: "stack-tasks", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & "/StackTasks.csv") - - # stack-users -t ../hayabusa/timeline.jsonl --level -o case-1/TargetUsers.csv - let cmd16 = StackUsersCmd(name: "stack-users(target)", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & - "/StackTargetUsers.csv", filterSystemAccounts: true, - filterComputerAccounts: true) - - # stack-users -t ../hayabusa/timeline.jsonl --level --filterSystemAccounts=false --filterComputerAccounts=false -o case-1/TargetUsers-NoFiltering.csv - let cmd17 = StackUsersCmd(name: "stack-users(no filtering)", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & - "/StackTargetUsers-NoFiltering.csv", filterSystemAccounts: false, - filterComputerAccounts: false) - - # stack-users -t ../hayabusa/timeline.jsonl --level --sourceUsers -o users.csv - let cmd18 = StackUsersCmd(name: "stack-users(source)", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & - "/StackSourceUsers.csv", sourceUsers: true, filterSystemAccounts: true, - filterComputerAccounts: true) - - # stack-users -t ../hayabusa/timeline.jsonl --level --sourceUsers --filterSystemAccounts=false --filterComputerAccounts=false -o case-1/SourceUsers-NoFiltering.csv - let cmd19 = StackUsersCmd(name: "stack-users(no filtering)", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & - "/StackSourceUsers-NoFiltering.csv", sourceUsers: true, - filterSystemAccounts: false, filterComputerAccounts: false) - - # timelinePath-logon -t ../hayabusa/timeline.jsonl -o case-1/LogonTimeline.csv - let cmd20 = TimelineLogonCmd(name: "timelinePath-logon", - displayTable: displayTable, timeline: timelinePath, output: output & "/TimelineLogon.csv") - - # timelinePath-partition-diagnostic -t ../hayabusa/timeline.jsonl -o case-1/PartitionDiagnosticTimeline.csv - let cmd21 = TimelinePartitionDiagnosticCmd( - name: "timelinePath-partition-diagnostic", displayTable: displayTable, - timeline: timelinePath, output: output & "/TimelinePartitionDiagnostic.csv") - - # timelinePath-suspicious-processes -t ../hayabusa/timeline.jsonl --level -o case-1/SuspiciousProcesses.csv - let cmd22 = TimelineSuspiciousProcessesCmd( - name: "timelinePath-suspicious-processes", level: level, - displayTable: displayTable, timeline: timelinePath, output: output & "/TimelineSuspiciousProcesses.csv") - - # timelinePath-tasks -t ../hayabusa/timeline.jsonl -o case-1/TaskTimeline.csv - let cmd23 = TimelineTasksCmd(name: "timelinePath-tasks", - displayTable: displayTable, timeline: timelinePath, output: output & "/TimelineTask.csv") - - # ttp-summary -t ../hayabusa/timeline.jsonl -o case-1/TTP-Summary.csv - let cmd24 = TTPSummaryCmd(name: "ttp-summary", displayTable: displayTable, - timeline: timelinePath, output: output & "/TTPSummary.csv") - cmd24.attack = readJsonFromFile("mitre-attack.json") - - # ttp-visualize -t ../hayabusa/timeline.jsonl -o case-1/MitreTTP-Heatmap.json - let cmd25 = TTPVisualizeCmd(name: "ttp-visualize", - displayTable: displayTable, timeline: timelinePath, output: output & "/MitreTTP-Heatmap.json") - - # execute all command - let cmds = @[cmd1, cmd2, cmd3, cmd4, cmd5, cmd6, cmd7, cmd8, cmd9, cmd10, - cmd11, cmd12, cmd13, cmd14, cmd15, cmd16, cmd17, cmd18, cmd19, - cmd20, - cmd21, cmd22, cmd23, cmd24, cmd25] - let cmd = AutoMagicCmd(level: level, skipProgressBar: skipProgressBar, - displayTable: displayTable, output: output, timeline: timelinePath, - name: "automagic", msg: AutoMagicMsg) - cmd.analyzeJSONLFile(cmds) + # extract-scriptblocks -t ../hayabusa/timeline.jsonl -l -o case-1/scriptblock-logs/ + let cmd1 = ExtractScriptBlocksCmd(name: "extract-scriptblocks", + level: level, skipProgressBar: skipProgressBar, + displayTable: displayTable, + timeline: timeline, output: output & "/scriptblock-logs") + if not dirExists(output & "/scriptblock-logs/"): + createDir(output & "/scriptblock-logs/") + + # list-domains -t ../hayabusa/timeline.jsonl -o case-1/ListDomains.txt + let cmd2 = ListDomainsCmd(name: "list-domains", displayTable: displayTable, + timeline: timeline, output: output & "/ListDomains.txt") + + # list-domains -t ../hayabusa/timeline.jsonl -d -w -o case-1/ListDomains-Detailed.txt + let cmd3 = ListDomainsCmd(name: "list-domains(detailed)", + displayTable: displayTable, timeline: timeline, output: output & + "/ListDomainsDetailed.txt", includeSubdomains: true, + includeWorkstations: true) + + # list-hashes -t ../hayabusa/timeline.jsonl -l -o case-1/hashes + let cmd4 = ListHashesCmd(name: "list-hashes", displayTable: displayTable, + timeline: timeline, level: level, output: output & "/ListHashes") + + # list-ip-addresses -t ../hayabusa/timeline.jsonl -o case-1/IP-Addresses.txt + let cmd5 = ListIpAddressesCmd(name: "list-ip-addresses", + displayTable: displayTable, timeline: timeline, output: output & "/ListIP-Addresses.txt") + + # stack-cmdlines -t ../hayabusa/timeline.jsonl --level -o case-1/cmdlines.csv + let cmd6 = StackCmdlineCmd(name: "stack-cmdlines", level: level, + displayTable: displayTable, timeline: timeline, output: output & "/StackCmdlines.csv") + + # stack-computers -t ../hayabusa/timeline.jsonl --level -o case-1/TargetComputers.csv + let cmd7 = StackComputersCmd(name: "stack-computers", level: level, + displayTable: displayTable, timeline: timeline, output: output & "/StackTargetComputers.csv") + + # stack-computers -t ../hayabusa/timeline.jsonl --level --sourceComputers -o case-1/SourceComputers.csv + let cmd8 = StackComputersCmd(name: "stack-computers", level: level, + displayTable: displayTable, timeline: timeline, output: output & + "/StackSourceComputers.csv", sourceComputers: true) + + # stack-dns -t ../hayabusa/timeline.jsonl --level -o case-1/DNS.csv + let cmd9 = StackDNSCmd(name: "stack-dns", level: level, + displayTable: displayTable, timeline: timeline, output: output & "/StackDNS.csv") + + # stack-ip-addresses -t ../hayabusa/timeline.jsonl --level -o case-1/SourceIP-Addresses.csv + let cmd10 = StackIpAddressesCmd(name: "stack-ip-addresses(source)", + level: level, displayTable: displayTable, timeline: timeline, + output: output & "/StackSourceIP-Addresses.csv") + + # stack-ip-addresses -t ../hayabusa/timeline.jsonl --level --targetIpAddresses -o case-1/TargetIP-Addresses.csv + let cmd11 = StackIpAddressesCmd(name: "stack-ip-addresses(target)", + level: level, displayTable: displayTable, timeline: timeline, + output: output & "/StackTargetIP-Addresses.csv", + targetIpAddresses: true) + + # stack-logons -t ../hayabusa/timeline.jsonl -o case-1/Logons.csv + let cmd12 = StackLogonsCmd(name: "stack-logons", displayTable: displayTable, + timeline: timeline, output: output & "/StackLogons.csv") + + # stack-processes -t ../hayabusa/timeline.jsonl --level -o case-1/Processes.csv + let cmd13 = StackProcessesCmd(name: "stack-processes", level: level, + displayTable: displayTable, timeline: timeline, output: output & "/StackProcesses.csv") + + # stack-services -t ../hayabusa/timeline.jsonl --level -o case-1/Services.csv + let cmd14 = StackServicesCmd(name: "stack-services", level: level, + displayTable: displayTable, timeline: timeline, output: output & "/StackServices.csv") + + # stack-tasks -t ../hayabusa/timeline.jsonl --level -o case-1/Tasks.csv + let cmd15 = StackTasksCmd(name: "stack-tasks", level: level, + displayTable: displayTable, timeline: timeline, output: output & "/StackTasks.csv") + + # stack-users -t ../hayabusa/timeline.jsonl --level -o case-1/TargetUsers.csv + let cmd16 = StackUsersCmd(name: "stack-users(target)", level: level, + displayTable: displayTable, timeline: timeline, output: output & + "/StackTargetUsers.csv", filterSystemAccounts: true, + filterComputerAccounts: true) + + # stack-users -t ../hayabusa/timeline.jsonl --level --filterSystemAccounts=false --filterComputerAccounts=false -o case-1/TargetUsers-NoFiltering.csv + let cmd17 = StackUsersCmd(name: "stack-users(no filtering)", level: level, + displayTable: displayTable, timeline: timeline, output: output & + "/StackTargetUsers-NoFiltering.csv", filterSystemAccounts: false, + filterComputerAccounts: false) + + # stack-users -t ../hayabusa/timeline.jsonl --level --sourceUsers -o users.csv + let cmd18 = StackUsersCmd(name: "stack-users(source)", level: level, + displayTable: displayTable, timeline: timeline, output: output & + "/StackSourceUsers.csv", sourceUsers: true, filterSystemAccounts: true, + filterComputerAccounts: true) + + # stack-users -t ../hayabusa/timeline.jsonl --level --sourceUsers --filterSystemAccounts=false --filterComputerAccounts=false -o case-1/SourceUsers-NoFiltering.csv + let cmd19 = StackUsersCmd(name: "stack-users(no filtering)", level: level, + displayTable: displayTable, timeline: timeline, output: output & + "/StackSourceUsers-NoFiltering.csv", sourceUsers: true, + filterSystemAccounts: false, filterComputerAccounts: false) + + # timeline-logon -t ../hayabusa/timeline.jsonl -o case-1/LogonTimeline.csv + let cmd20 = TimelineLogonCmd(name: "timeline-logon", + displayTable: displayTable, timeline: timeline, output: output & "/TimelineLogon.csv") + + # timeline-partition-diagnostic -t ../hayabusa/timeline.jsonl -o case-1/PartitionDiagnosticTimeline.csv + let cmd21 = TimelinePartitionDiagnosticCmd( + name: "timeline-partition-diagnostic", displayTable: displayTable, + timeline: timeline, output: output & "/TimelinePartitionDiagnostic.csv") + + # timeline-suspicious-processes -t ../hayabusa/timeline.jsonl --level -o case-1/SuspiciousProcesses.csv + let cmd22 = TimelineSuspiciousProcessesCmd( + name: "timeline-suspicious-processes", level: level, + displayTable: displayTable, timeline: timeline, output: output & "/TimelineSuspiciousProcesses.csv") + + # timeline-tasks -t ../hayabusa/timeline.jsonl -o case-1/TaskTimeline.csv + let cmd23 = TimelineTasksCmd(name: "timeline-tasks", + displayTable: displayTable, timeline: timeline, output: output & "/TimelineTask.csv") + + # ttp-summary -t ../hayabusa/timeline.jsonl -o case-1/TTP-Summary.csv + let cmd24 = TTPSummaryCmd(name: "ttp-summary", displayTable: displayTable, + timeline: timeline, output: output & "/TTPSummary.csv") + cmd24.attack = readJsonFromFile("mitre-attack.json") + + # ttp-visualize -t ../hayabusa/timeline.jsonl -o case-1/MitreTTP-Heatmap.json + let cmd25 = TTPVisualizeCmd(name: "ttp-visualize", + displayTable: displayTable, timeline: timeline, output: output & "/MitreTTP-Heatmap.json") + + # execute all command + let cmds = @[cmd1, cmd2, cmd3, cmd4, cmd5, cmd6, cmd7, cmd8, cmd9, cmd10, + cmd11, cmd12, cmd13, cmd14, cmd15, cmd16, cmd17, cmd18, cmd19, + cmd20, + cmd21, cmd22, cmd23, cmd24, cmd25] + let cmd = AutoMagicCmd(level: level, skipProgressBar: skipProgressBar, + displayTable: displayTable, output: output, timeline: timeline, + name: "automagic", msg: AutoMagicMsg) + cmd.analyzeJSONLFile(cmds) diff --git a/src/takajopkg/extractScriptblocks.nim b/src/takajopkg/extractScriptblocks.nim index ae1a83a6..62f349a3 100644 --- a/src/takajopkg/extractScriptblocks.nim +++ b/src/takajopkg/extractScriptblocks.nim @@ -17,7 +17,7 @@ proc outputScriptText(output: string, timestamp: string, computerName: string, let date = timestamp.replace(":", "_").replace(" ", "_") let fileName = output & "/" & computerName & "-" & date & "-" & scriptObj.scriptBlockId & ".txt" - var outputFile = open(filename, fmAppend) + var outputFile = open(filename, fmWrite) outputFile.write(scriptText) flushFile(outputFile) close(outputFile) @@ -110,7 +110,7 @@ method resultOutput*(self: ExtractScriptBlocksCmd) = let summaryFile = self.output & "/" & "Summary.csv" let header = ["Creation Time", "Computer Name", "Script ID", "Script Name", "Records", "Level", "Alerts"] - var outputFile = open(summaryFile, fmAppend) + var outputFile = open(summaryFile, fmWrite) var table: TerminalTable table.add header for i, val in header: @@ -159,5 +159,5 @@ proc extractScriptblocks(level: string = "low", skipProgressBar: bool = false, output: output, name: "extract-scriptblocks", msg: ExtractScriptBlocksMsg) - cmd.totalLines = countLinesInTimeline(timeline, true) + cmd.totalLines += countLinesInTimeline(timelinePath, true) cmd.analyzeJSONLFile() diff --git a/src/takajopkg/general.nim b/src/takajopkg/general.nim index fc6bed8f..1b8133e8 100644 --- a/src/takajopkg/general.nim +++ b/src/takajopkg/general.nim @@ -243,26 +243,32 @@ proc formatFileSize*(fileSize: BiggestInt): string = return fileSizeStr proc countLinesInTimeline*(filePath: string, quiet: bool = false): int = - echo "File: " & filePath & " (" & formatFileSize(getFileSize(filePath)) & ")" + let fileInfo = getFileInfo(filePath) + var filePaths = @[filePath] + if (fileInfo.kind == pcDir or fileInfo.kind == pcLinkToDir): + filePaths = getTargetExtFileLists(filePath, ".jsonl", true) + var count = 0 if not quiet: echo "Counting total lines. Please wait." - const BufferSize = 4 * 1024 * 1024 # 4 MiB - var buffer = newString(BufferSize) - var file = open(filePath) - var count = 0 - - while true: - let bytesRead = file.readChars(buffer.toOpenArray(0, BufferSize - 1)) - if bytesRead == 0: - break - for i in 0 ..< bytesRead: - if buffer[i] == '\n': - inc(count) - inc(count) - file.close() + for path in filePaths: + echo "File: " & path & " (" & formatFileSize(getFileSize(path)) & ")" + const BufferSize = 4 * 1024 * 1024 # 4 MiB + var buffer = newString(BufferSize) + var file = open(path) + + while true: + let bytesRead = file.readChars(buffer.toOpenArray(0, BufferSize - 1)) + if bytesRead == 0: + break + for i in 0 ..< bytesRead: + if buffer[i] == '\n': + inc(count) + inc(count) + file.close() if not quiet: echo "Total lines: ", intToStr(count).insertSep(',') echo "" + return count proc isMinLevel*(levelInLog: string, userSetLevel: string): bool = diff --git a/src/takajopkg/listDomains.nim b/src/takajopkg/listDomains.nim index 35bdf674..e03b2d61 100644 --- a/src/takajopkg/listDomains.nim +++ b/src/takajopkg/listDomains.nim @@ -29,7 +29,7 @@ method analyze*(self: ListDomainsCmd, x: HayabusaJson) = method resultOutput*(self: ListDomainsCmd) = # Save results - var outputFile = open(self.output, fmAppend) + var outputFile = open(self.output, fmWrite) for domain in self.domainHashSet: outputFile.write(domain & "\p") let outputFileSize = getFileSize(outputFile) diff --git a/src/takajopkg/splitCsvTimeline.nim b/src/takajopkg/splitCsvTimeline.nim index 2c0717d8..a4c2ac88 100644 --- a/src/takajopkg/splitCsvTimeline.nim +++ b/src/takajopkg/splitCsvTimeline.nim @@ -66,7 +66,7 @@ proc splitCsvTimeline(makeMultiline: bool = false, output: string = "output", if not filesTable.hasKey(computerName): let filename = output & "/" & computerName & "-HayabusaResults.csv" filenameSequence.add(filename) - var outputFile = open(filename, fmAppend) + var outputFile = open(filename, fmWrite) filesTable[computerName] = outputFile outputFile.write(csvHeader) outputFile.write("\p") diff --git a/src/takajopkg/takajoCore.nim b/src/takajopkg/takajoCore.nim index 4ee50dc1..f143525f 100644 --- a/src/takajopkg/takajoCore.nim +++ b/src/takajopkg/takajoCore.nim @@ -6,21 +6,22 @@ import std/enumerate import std/options import suru import times +import std/os type CmdResult* = ref object - results*: string = "" - savedFiles*: string = "" + results*: string = "" + savedFiles*: string = "" type AbstractCmd* = ref object of RootObj - timeline*: string = "" - skipProgressBar*: bool - name*: string = "" - msg*: string = "" - output*: string = "" - cmdResult*: CmdResult = CmdResult(results:"", savedFiles:"") - displayTable*: bool = true + timeline*: string = "" + skipProgressBar*: bool + name*: string = "" + msg*: string = "" + output*: string = "" + cmdResult*: CmdResult = CmdResult(results: "", savedFiles: "") + displayTable*: bool = true -method filter*(self: AbstractCmd, x: HayabusaJson):bool {.base.} = +method filter*(self: AbstractCmd, x: HayabusaJson): bool {.base.} = return true method analyze*(self: AbstractCmd, x: HayabusaJson) {.base.} = @@ -29,49 +30,55 @@ method analyze*(self: AbstractCmd, x: HayabusaJson) {.base.} = method resultOutput*(self: AbstractCmd) {.base.} = raise newException(ValueError, "resultOutput(AbstractCmd) is not implemented") -proc analyzeJSONLFile*(self: AbstractCmd, cmds: seq[AbstractCmd] = newSeq[AbstractCmd]()) = - let startTime = epochTime() - var commands = cmds - if commands.len() == 0: - # automagic以外では、自身のオブジェクトだけを実行 - commands = @[self] +proc analyzeJSONLFile*(self: AbstractCmd, cmds: seq[AbstractCmd] = newSeq[ + AbstractCmd]()) = + let startTime = epochTime() + var commands = cmds + if commands.len() == 0: + # automagic以外では、自身のオブジェクトだけを実行 + commands = @[self] - var bar: SuruBar - if not self.skipProgressBar: - bar = initSuruBar() - bar[0].total = countJsonlAndStartMsg(self.name, self.msg, self.timeline) - bar.setup() + var bar: SuruBar + if not self.skipProgressBar: + bar = initSuruBar() + bar[0].total = countJsonlAndStartMsg(self.name, self.msg, self.timeline) + bar.setup() - for line in lines(self.timeline): - if not self.skipProgressBar: - inc bar - bar.update(1000000000) - let jsonLineOpt = parseLine(line) - if jsonLineOpt.isNone: - continue - let jsonLine:HayabusaJson = jsonLineOpt.get() - for cmd in commands: - try: - if cmd.filter(jsonLine): - cmd.analyze(jsonLine) - except CatchableError: - continue + let fileInfo = getFileInfo(self.timeline) + var filePaths = @[self.timeline] + if (fileInfo.kind == pcDir or fileInfo.kind == pcLinkToDir): + filePaths = getTargetExtFileLists(self.timeline, ".jsonl", true) + for path in filePaths: + for line in lines(path): + if not self.skipProgressBar: + inc bar + bar.update(1000000000) + let jsonLineOpt = parseLine(line) + if jsonLineOpt.isNone: + continue + let jsonLine: HayabusaJson = jsonLineOpt.get() + for cmd in commands: + try: + if cmd.filter(jsonLine): + cmd.analyze(jsonLine) + except CatchableError: + continue - if not self.skipProgressBar: - bar.finish() + if not self.skipProgressBar: + bar.finish() - var resultSummaryTable:seq[CmdResult] = @[] - for cmd in commands: - cmd.resultOutput() - resultSummaryTable.add(cmd.cmdResult) + var resultSummaryTable: seq[CmdResult] = @[] + for cmd in commands: + cmd.resultOutput() + resultSummaryTable.add(cmd.cmdResult) - if resultSummaryTable.len > 1: - # automagicコマンドの時だけ、最後に全コマンドまとめたサマリを出力する - var table: TerminalTable - table.add @["Command", "Results", "Saved Files"] - for i, result in enumerate(resultSummaryTable): - table.add commands[i].name, result.results, result.savedFiles - echo "" - table.echoTableSepsWithStyled(seps = boxSeps) + if resultSummaryTable.len > 1: + # automagicコマンドの時だけ、最後に全コマンドまとめたサマリを出力する + var table: TerminalTable + table.add @["Command", "Results", "Saved Files"] + for i, result in enumerate(resultSummaryTable): + table.add commands[i].name, result.results, result.savedFiles + echo "" + table.echoTableSepsWithStyled(seps = boxSeps) - outputElapsedTime(startTime) \ No newline at end of file + outputElapsedTime(startTime) diff --git a/src/takajopkg/timelineTasks.nim b/src/takajopkg/timelineTasks.nim index 00f930c3..e55de6ed 100644 --- a/src/takajopkg/timelineTasks.nim +++ b/src/takajopkg/timelineTasks.nim @@ -74,7 +74,7 @@ method resultOutput*(self: TimelineTasksCmd) = let detailedHeader = toSeq(allDetailedKeys).sorted # Save results - var outputFile = open(self.output, fmAppend) + var outputFile = open(self.output, fmWrite) ## Write CSV header let header = concat(basicHeader, detailedHeader) diff --git a/src/takajopkg/ttpResult.nim b/src/takajopkg/ttpResult.nim index 56fc66f9..6ee7975e 100644 --- a/src/takajopkg/ttpResult.nim +++ b/src/takajopkg/ttpResult.nim @@ -58,7 +58,7 @@ proc outputTTPResult*(stackedMitreTags: Table[string, string], "selectVisibleTechniques": false } - let outputFile = open(output, FileMode.fmAppend) + let outputFile = open(output, FileMode.fmWrite) outputFile.write(jsonObj.pretty()) let outputFileSize = getFileSize(outputFile) savedFiles = output & " (" & formatFileSize(outputFileSize) & ")" diff --git a/src/takajopkg/ttpSummary.nim b/src/takajopkg/ttpSummary.nim index 5f3db0e0..8a9d6cd5 100644 --- a/src/takajopkg/ttpSummary.nim +++ b/src/takajopkg/ttpSummary.nim @@ -67,7 +67,7 @@ method resultOutput*(self: TTPSummaryCmd) = var ruleStr = initHashSet[string]() if self.output != "": # Open file to save results - var outputFile = open(self.output, fmAppend) + var outputFile = open(self.output, fmWrite) ## Write CSV header outputFile.write(header.join(",") & "\p") From 2251ac0bc46668c2c75a30307ffcd797b02bf918 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 29 Mar 2024 02:28:24 +0900 Subject: [PATCH 6/9] fix: fixed bug split-csv-timeline caused by directory processing #133 --- src/takajopkg/general.nim | 12 +++++++----- src/takajopkg/splitCsvTimeline.nim | 7 ++----- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/src/takajopkg/general.nim b/src/takajopkg/general.nim index 1b8133e8..64f2f84b 100644 --- a/src/takajopkg/general.nim +++ b/src/takajopkg/general.nim @@ -242,11 +242,12 @@ proc formatFileSize*(fileSize: BiggestInt): string = fileSizeStr = $fileSize & " Bytes" return fileSizeStr -proc countLinesInTimeline*(filePath: string, quiet: bool = false): int = +proc countLinesInTimeline*(filePath: string, quiet: bool = false, + ext: string = ".jsonl"): int = let fileInfo = getFileInfo(filePath) var filePaths = @[filePath] if (fileInfo.kind == pcDir or fileInfo.kind == pcLinkToDir): - filePaths = getTargetExtFileLists(filePath, ".jsonl", true) + filePaths = getTargetExtFileLists(filePath, ext, true) var count = 0 if not quiet: echo "Counting total lines. Please wait." @@ -376,7 +377,8 @@ proc outputElapsedTime*(startTime: float) = $seconds & " seconds" echo "" -proc checkArgs*(quiet: bool = false, timeline: string, level: string) = +proc checkArgs*(quiet: bool = false, timeline: string, level: string, + ext: string = ".jsonl") = if not quiet: styledEcho(fgGreen, outputLogo()) @@ -388,9 +390,9 @@ proc checkArgs*(quiet: bool = false, timeline: string, level: string) = echo "The file '" & timeline & "' does not exist. Please specify a valid file path." quit(1) - var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) + var filePaths = getTargetExtFileLists(timeline, ext, true) for timelinePath in filePaths: - if not isJsonConvertible(timelinePath): + if ext == ".jsonl" and not isJsonConvertible(timelinePath): quit(1) if level != "critical" and level != "high" and level != "medium" and diff --git a/src/takajopkg/splitCsvTimeline.nim b/src/takajopkg/splitCsvTimeline.nim index a4c2ac88..85f5c03a 100644 --- a/src/takajopkg/splitCsvTimeline.nim +++ b/src/takajopkg/splitCsvTimeline.nim @@ -1,10 +1,7 @@ proc splitCsvTimeline(makeMultiline: bool = false, output: string = "output", quiet: bool = false, timeline: string) = let startTime = epochTime() - if not quiet: - styledEcho(fgGreen, outputLogo()) - - checkArgs(quiet, timeline, "informational") + checkArgs(quiet, timeline, "informational", ext = ".csv") echo "Started the Split CSV Timeline command" echo "" @@ -15,7 +12,7 @@ proc splitCsvTimeline(makeMultiline: bool = false, output: string = "output", var totalLines = 0 var filePaths = getTargetExtFileLists(timeline, ".csv", true) for timelinePath in filePaths: - totalLines += countLinesInTimeline(timelinePath) + totalLines += countLinesInTimeline(timelinePath, quiet, ".csv") echo "Splitting the Hayabusa CSV timeline. Please wait." From 1741ef59c5d8b2928f92e3a8c40cc389389514b1 Mon Sep 17 00:00:00 2001 From: DastInDark <2350416+hitenkoku@users.noreply.github.com> Date: Fri, 29 Mar 2024 02:35:53 +0900 Subject: [PATCH 7/9] fix: fixed bug caused by selected wrong extension #133 --- src/takajopkg/splitJsonTimeline.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/takajopkg/splitJsonTimeline.nim b/src/takajopkg/splitJsonTimeline.nim index b64d298e..1ab404c5 100644 --- a/src/takajopkg/splitJsonTimeline.nim +++ b/src/takajopkg/splitJsonTimeline.nim @@ -9,7 +9,7 @@ proc splitJsonTimeline(output: string = "output", quiet: bool = false, echo "" var totalLines = 0 - var filePaths = getTargetExtFileLists(timeline, ".csv", true) + var filePaths = getTargetExtFileLists(timeline, ".jsonl", true) for timelinePath in filePaths: totalLines += countLinesInTimeline(timelinePath) From 636cbcd1d05249f0911442c402d6016e691fdf9e Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Fri, 29 Mar 2024 09:49:06 +0900 Subject: [PATCH 8/9] updated changelog --- CHANGELOG-Japanese.md | 1 + CHANGELOG.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG-Japanese.md b/CHANGELOG-Japanese.md index 93f29123..6eae2cd5 100644 --- a/CHANGELOG-Japanese.md +++ b/CHANGELOG-Japanese.md @@ -7,6 +7,7 @@ - `stack-computers`コマンド: `Computer`(デフォルト)または`SrcComp`フィールドのスタック分析をしながら、アラート情報も提供する。(#125) (@fukusuket) - `stack-ip-addresses`コマンド: `SrcIP`(デフォルト)または`TgtIP`フィールドのスタック分析をしながら、アラート情報も提供する。(#129) (@fukusuket) - `stack-users`コマンド: `TgtUser`(デフォルト)または`SrcUser`フィールドのスタック分析をしながら、アラート情報も提供する。(#130) (@fukusuket) +- スキャン時に複数の`.jsonl`ファイルが入っているディレクトリを指定できるようになった。 #133 (@hitenkoku) **改善:** diff --git a/CHANGELOG.md b/CHANGELOG.md index 70e04585..671032b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - `stack-computers` command: stack the `Computer` (default) or `SrcComp` fields as well as provide alert information. (#125) (@fukusuket) - `stack-ip-addresses` command: stack the `SrcIP` (default) or `TgtIP` fields as well as provide alert information. (#129) (@fukusuket) - `stack-users` command: stack the `TgtUser` (default) or `SrcUser` fields as well as provide alert information. (#130) (@fukusuket) +- You can now specify a directory of `.jsonl` files to scan. #133 (@hitenkoku) **Enhancements:** From 5c2dd77e59894702fac8719e2fb65e43c7e5d210 Mon Sep 17 00:00:00 2001 From: Yamato Security <71482215+YamatoSecurity@users.noreply.github.com> Date: Fri, 29 Mar 2024 09:51:33 +0900 Subject: [PATCH 9/9] update wording --- README.md | 10 +++++----- src/takajo.nim | 44 ++++++++++++++++++++++---------------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/README.md b/README.md index ee3c4ae8..e28304ae 100644 --- a/README.md +++ b/README.md @@ -106,7 +106,7 @@ Takajō means ["Falconer"](https://en.wikipedia.org/wiki/Falconry) in Japanese a - [`ttp-visualize` command examples](#ttp-visualize-command-examples) - [`ttp-visualize` screenshot](#ttp-visualize-screenshot) - [`ttp-visualize-sigma` command](#ttp-visualize-sigma-command) - - [`ttp-visualize-simga` command examples](#ttp-visualize-sigma-command-examples) + - [`ttp-visualize-sigma` command examples](#ttp-visualize-sigma-command-examples) - [VirusTotal Commands](#virustotal-commands-1) - [`vt-domain-lookup` command](#vt-domain-lookup-command) - [`vt-domain-lookup` command examples](#vt-domain-lookup-command-examples) @@ -212,7 +212,7 @@ Extracts and reassemles PowerShell EID 4104 script block logs. Required options: -- `-t, --timeline `: Hayabusa JSONL timeline +- `-t, --timeline `: Hayabusa JSONL timeline file or directory Options: @@ -252,7 +252,7 @@ Currently it will only check queried domains in Sysmon EID 22 logs but will be u Required options: - `-o, --output `: save results to a text file. -- `-t, --timeline `: Hayabusa JSONL timeline. +- `-t, --timeline `: Hayabusa JSONL timeline file or directory. Options: @@ -326,7 +326,7 @@ It will extract the `TgtIP` fields for target IP addresses and `SrcIP` fields fo Required options: - `-o, --output `: save results to a text file. -- `-t, --timeline `: Hayabusa JSONL timeline. +- `-t, --timeline `: Hayabusa JSONL timeline file or directory. Options: @@ -496,7 +496,7 @@ Split up a large JSONL timeline into smaller ones based on the computer name. Required options: -- `-t, --timeline `: Hayabusa JSONL timeline. +- `-t, --timeline `: Hayabusa JSONL timeline file or directory. Options: diff --git a/src/takajo.nim b/src/takajo.nim index 174dda96..570e32e5 100644 --- a/src/takajo.nim +++ b/src/takajo.nim @@ -114,7 +114,7 @@ when isMainModule: "displayTable": "display the result table", "output": "output directory (default: scriptblock-logs)", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any)", } ], [ @@ -125,7 +125,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "output directory (default: scriptblock-logs)", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any)", } ], [ @@ -137,7 +137,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a text file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", }, short = { "includeSubdomains": 'd', @@ -152,7 +152,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "specify the base name to save results to text files (ex: -o case-1)", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", } ], [ @@ -165,7 +165,7 @@ when isMainModule: "output": "save results to a text file", "privateIp": "include private IP addresses", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", }, short = { "output": 'o', @@ -210,7 +210,7 @@ when isMainModule: help = { "output": "output directory (default: output)", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any)", } ], [ @@ -222,7 +222,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", }, short = { "sourceComputers": 'c' @@ -238,7 +238,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", }, short = { "ignoreSysmon": 'y', @@ -253,7 +253,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", } ], [ @@ -265,7 +265,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any)", }, short = { "targetIpAddresses": 'a' @@ -279,7 +279,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", } ], [ @@ -292,7 +292,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", }, short = { "ignoreSysmon": 'y', @@ -307,7 +307,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", }, short = { "ignoreSystem": 'y', @@ -322,7 +322,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", } ], [ @@ -336,7 +336,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any)", }, short = { "filterComputerAccounts": 'c', @@ -350,7 +350,7 @@ when isMainModule: "output": "save results to a text file", "processGuid": "sysmon process GUID", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", } ], [ @@ -363,7 +363,7 @@ when isMainModule: "outputLogoffEvents": "output logoff events as separate entries", "skipProgressBar": "do not display the progress bar", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", }, short = { "outputLogoffEvents": 'l', @@ -377,7 +377,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any)", } ], [ @@ -388,7 +388,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any besides all-field-info*)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any besides all-field-info*)", } ], [ @@ -398,7 +398,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a CSV file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any)", } ], [ @@ -408,7 +408,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a csv file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any verbose profile)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any verbose profile)", } ], [ @@ -418,7 +418,7 @@ when isMainModule: "skipProgressBar": "do not display the progress bar", "output": "save results to a json file", "quiet": "do not display the launch banner", - "timeline": "Hayabusa JSONL timeline (profile: any verbose profile)", + "timeline": "Hayabusa JSONL timeline file or directory (profile: any verbose profile)", } ], [