Copy over local package sources (#4300)

packages/cisco_secure_email_gateway/0.1.0/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.1.0"
+ changes:
+ - description: Initial draft of the package
+ type: enhancement
+ link: https:/elastic/integrations/pull/3040

packages/cisco_secure_email_gateway/0.1.0/data_stream/log/agent/stream/stream.yml.hbs

@@ -0,0 +1,19 @@
+paths:
+{{#each paths as |path i|}}
+ - {{path}}
+{{/each}}
+tags:
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+ - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+exclude_files: [".gz$"]
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}

packages/cisco_secure_email_gateway/0.1.0/data_stream/log/agent/stream/tcp.yml.hbs

@@ -0,0 +1,19 @@
+host: "{{listen_address}}:{{listen_port}}"
+tags:
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+ - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if ssl}}
+ssl: {{ssl}}
+{{/if}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
+

packages/cisco_secure_email_gateway/0.1.0/data_stream/log/agent/stream/udp.yml.hbs

@@ -0,0 +1,16 @@
+host: "{{listen_address}}:{{listen_port}}"
+tags:
+{{#if preserve_original_event}}
+ - preserve_original_event
+{{/if}}
+{{#each tags as |tag|}}
+ - {{tag}}
+{{/each}}
+{{#contains "forwarded" tags}}
+publisher_pipeline.disable_host: true
+{{/contains}}
+{{#if processors}}
+processors:
+{{processors}}
+{{/if}}
+

packages/cisco_secure_email_gateway/0.1.0/data_stream/log/elasticsearch/ingest_pipeline/default.yml

@@ -0,0 +1,103 @@
+---
+description: Pipeline for Cisco Secure Email Gateway logs
+processors:
+ - set:
+ field: ecs.version
+ value: "8.2.0"
+ - set:
+ field: _tmp.filepath
+ value: "{{{log.file.path}}}"
+ if: ctx?.log?.file?.path != null
+ - grok:
+ field: _tmp.filepath
+ if: ctx?.log?.file?.path != null
+ patterns:
+ - "^%{DATA}/%{WORD:cisco_secure_email_gateway.log.category.name}.@%{GREEDYDATA}.s"
+ - rename:
+ field: message
+ target_field: event.original
+ ignore_missing: true
+ - grok:
+ field: event.original
+ patterns:
+ - "^<%{NUMBER:log.syslog.priority:long}>%{SYSLOGTIMESTAMP:_tmp.timestamp} %{DATA:cisco_secure_email_gateway.log.category.name}: %{WORD:log.level}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}"
+ - "^<%{NUMBER:log.syslog.priority:long}>%{SYSLOGTIMESTAMP:_tmp.timestamp} %{DATA:cisco_secure_email_gateway.log.category.name}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}"
+ - "^%{DATA:_tmp.timestamp} %{WORD:log.level}: %{GREEDYDATA:cisco_secure_email_gateway.log.message}"
+ - "^%{GREEDYDATA:cisco_secure_email_gateway.log.message}"
+ - lowercase:
+ field: log.level
+ ignore_failure: true
+ - date:
+ field: _tmp.timestamp
+ target_field: "@timestamp"
+ formats:
+ - E MMM dd HH:mm:ss yyyy
+ - E MMM d HH:mm:ss yyyy
+ - E MMM d HH:mm:ss yyyy
+ - MMM d HH:mm:ss
+ - MMM dd HH:mm:ss
+ - MMM d HH:mm:ss
+ ignore_failure: true
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_authentication" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "authentication"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_gui_logs" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "gui_logs"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_anti_spam" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "antispam"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_error_logs" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "error_logs"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_text_mail_logs" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "mail_logs"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_content_scanner" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "content_scanner"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_system" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "system"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_bounce" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "bounces"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_status" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "status"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_amp" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "amp"
+ - pipeline:
+ name: '{{ IngestPipeline "pipeline_consolidated_event" }}'
+ if: ctx?.cisco_secure_email_gateway?.log?.category?.name == "consolidated_event"
+ - remove:
+ field:
+ - _tmp
+ ignore_missing: true
+ - remove:
+ field: event.original
+ if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
+ ignore_failure: true
+ ignore_missing: true
+ - script:
+ lang: painless
+ source: 
+ boolean dropEmptyFields(Object object) {
+ if (object == null || object == "") {
+ return true;
+ } else if (object instanceof Map) {
+ ((Map) object).values().removeIf(value -> dropEmptyFields(value));
+ return (((Map) object).size() == 0);
+ } else if (object instanceof List) {
+ ((List) object).removeIf(value -> dropEmptyFields(value));
+ return (((List) object).length == 0);
+ }
+ return false;
+ }
+ dropEmptyFields(ctx);
+on_failure:
+- set:
+ field: error.message
+ value: "{{{ _ingest.on_failure_message }}}"
+ 

packages/cisco_secure_email_gateway/0.1.0/data_stream/log/elasticsearch/ingest_pipeline/pipeline_amp.yml

@@ -0,0 +1,122 @@
+---
+processors:
+ - set:
+ field: event.kind
+ value: event
+ - grok:
+ field: cisco_secure_email_gateway.log.message
+ patterns:
+ - "^File reputation query initiating. %{GREEDYDATA:_tmp.new_message}"
+ - "^Response received for file reputation query from Cloud. %{GREEDYDATA:_tmp.new_message}"
+ - "^File Analysis complete. SHA256: %{GREEDYDATA:email.attachments.file.hash.sha256}, Submit Timestamp: %{GREEDYDATA:_tmp.submit.timestamp}, Update Timestamp: %{GREEDYDATA:_tmp.update.timestamp}, Disposition: %{DATA:cisco_secure_email_gateway.log.disposition} Score: %{NUMBER:cisco_secure_email_gateway.log.score:long}, run_id: %{NUMBER:cisco_secure_email_gateway.log.run_id} Details: %{DATA:cisco_secure_email_gateway.log.details} Spyname:\\[%{GREEDYDATA:cisco_secure_email_gateway.log.spy_name}\\]"
+ - "^File not uploaded for analysis. MID = %{NUMBER:email.message_id} File SHA256\\[%{GREEDYDATA:email.attachments.file.hash.sha256}\\] file mime\\[%{GREEDYDATA:email.attachments.file.mime_type}\\] Reason: %{GREEDYDATA:event.reason}"
+ - "^File analysis upload skipped. SHA256: %{GREEDYDATA:email.attachments.file.hash.sha256},Timestamp\\[%{GREEDYDATA:_tmp.submit.timestamp}\\] details\\[%{GREEDYDATA:_tmp.cisco_secure_email_gateway.log.remaining_details}]"
+ - "^SHA256: %{GREEDYDATA:email.attachments.file.hash.sha256},Timestamp\\[%{GREEDYDATA:_tmp.submit.timestamp}\\] details\\[%{GREEDYDATA:cisco_secure_email_gateway.log.server_error_details}\\]"
+ - "^Retrospective verdict received. %{GREEDYDATA:_tmp.new_message}" 
+ - "^%{GREEDYDATA:cisco_secure_email_gateway.log.message}"
+ - kv:
+ field: _tmp.new_message
+ if: ctx._tmp.new_message != null
+ field_split: ", |,"
+ value_split: " = | =|: "
+ - grok:
+ field: _tmp.cisco_secure_email_gateway.log.remaining_details
+ if: ctx?._tmp?.cisco_secure_email_gateway?.log?.remaining_details != null
+ patterns:
+ - "^File SHA256\\[%{GREEDYDATA:email.attachments.file.hash.sha256}\\] file mime\\[%{GREEDYDATA:email.attachments.file.mime_type}\\], upload priority\\[%{GREEDYDATA:cisco_secure_email_gateway.log.upload.priority}\\] not uploaded, re-tries\\[%{GREEDYDATA:cisco_secure_email_gateway.log.retries:long}\\], backoff\\[%{GREEDYDATA:cisco_secure_email_gateway.log.backoff:long}\\] %{GREEDYDATA:cisco_secure_email_gateway.log.details}"
+ - rename:
+ field: "Timestamp"
+ target_field: _tmp.submit.timestamp
+ ignore_missing: true
+ - date: 
+ field: _tmp.submit.timestamp
+ target_field: cisco_secure_email_gateway.log.submit.timestamp
+ if: ctx?.cisco_secure_email_gateway?.log?._tmp?.submit?.timestamp != "0"
+ ignore_failure: true
+ formats: 
+ - UNIX
+ - date: 
+ field: _tmp.update.timestamp
+ target_field: cisco_secure_email_gateway.log.update.timestamp
+ if: ctx?.cisco_secure_email_gateway?.log?._tmp?.update?.timestamp != "0"
+ ignore_failure: true
+ formats: 
+ - UNIX 
+ - rename:
+ field: "File Name"
+ target_field: email.attachments.file.name
+ ignore_missing: true
+ - rename:
+ field: "MID"
+ target_field: email.message_id
+ ignore_missing: true
+ - gsub:
+ field: "File Size"
+ pattern: "\ bytes"
+ replacement: ""
+ ignore_failure: true
+ - convert:
+ field: "File Size"
+ target_field: email.attachments.file.size
+ type: long
+ ignore_missing: true
+ - rename:
+ field: "File Type"
+ target_field: email.content_type
+ ignore_missing: true
+ - rename:
+ field: "FileName"
+ target_field: email.attachments.file.name
+ ignore_missing: true
+ - rename:
+ field: "Malware"
+ target_field: cisco_secure_email_gateway.log.malware
+ ignore_missing: true
+ - rename:
+ field: "Disposition"
+ target_field: cisco_secure_email_gateway.log.disposition
+ ignore_missing: true
+ - rename:
+ field: "Reputation Score"
+ target_field: cisco_secure_email_gateway.log.reputation_score
+ ignore_missing: true
+ - rename:
+ field: "sha256"
+ target_field: email.attachments.file.hash.sha256
+ ignore_missing: true
+ - rename:
+ field: "upload_action"
+ target_field: cisco_secure_email_gateway.log.upload.action
+ ignore_missing: true
+ - rename:
+ field: "Reputation Score"
+ target_field: cisco_secure_email_gateway.log.reputation_score
+ ignore_missing: true
+ - rename:
+ field: "SHA256"
+ target_field: email.attachments.file.hash.sha256
+ ignore_missing: true
+ - rename:
+ field: "Spyname"
+ target_field: cisco_secure_email_gateway.log.spy_name
+ ignore_missing: true
+ - rename:
+ field: "Verdict"
+ target_field: cisco_secure_email_gateway.log.verdict
+ ignore_missing: true
+ - gsub:
+ field: email.attachments.file.name
+ pattern: \'
+ replacement: ""
+ ignore_failure: true
+ - append:
+ field: related.hash
+ value: "{{{email.attachments.file.hash.sha256}}}"
+ if: ctx?.email?.attachments?.file?.hash?.sha256 != null
+ allow_duplicates: false
+ ignore_failure: true
+ - remove:
+ field: 
+ - _tmp
+ - "File Size"
+ ignore_missing: true

packages/cisco_secure_email_gateway/0.1.0/data_stream/log/elasticsearch/ingest_pipeline/pipeline_anti_spam.yml

@@ -0,0 +1,12 @@
+---
+processors:
+ - set:
+ field: event.kind
+ value: event
+ - grok:
+ field: cisco_secure_email_gateway.log.message
+ patterns:
+ - "^case %{GREEDYDATA:cisco_secure_email_gateway.log.object_category} \\(%{NUMBER:cisco_secure_email_gateway.log.case_id}\\) : case-daemon: all %{DATA:cisco_secure_email_gateway.log.object} killed, %{GREEDYDATA:cisco_secure_email_gateway.log.result}"
+ - "^case %{GREEDYDATA:cisco_secure_email_gateway.log.object_category} \\(%{NUMBER:cisco_secure_email_gateway.log.case_id}\\) : case-daemon: %{DATA:cisco_secure_email_gateway.log.object} killed by %{DATA:cisco_secure_email_gateway.log.command}, %{GREEDYDATA:cisco_secure_email_gateway.log.result}"
+ - "^case %{GREEDYDATA:cisco_secure_email_gateway.log.object_category} \\(%{NUMBER:cisco_secure_email_gateway.log.case_id}\\) : case-daemon: %{GREEDYDATA:cisco_secure_email_gateway.log.result}"
+ - "^%{GREEDYDATA:cisco_secure_email_gateway.log.message}"

packages/cisco_secure_email_gateway/0.1.0/data_stream/log/elasticsearch/ingest_pipeline/pipeline_authentication.yml

@@ -0,0 +1,51 @@
+---
+processors:
+ - set:
+ field: event.kind
+ value: event
+ - set:
+ field: event.category
+ value: [authentication]
+ - grok:
+ field: cisco_secure_email_gateway.log.message
+ patterns:
+ - "^GUI: User %{USERNAME:user.name} %{GREEDYDATA:cisco_secure_email_gateway.log.action} from session %{GREEDYDATA:cisco_secure_email_gateway.log.session} because of inactivity timeout"
+ - "^CLI: User %{USERNAME:user.name} %{GREEDYDATA:cisco_secure_email_gateway.log.action} from %{GREEDYDATA:cisco_secure_email_gateway.log.session} because of inactivity timeout"
+ - "^%{WORD:cisco_secure_email_gateway.log.action}:%{IP:host.ip} user:%{USERNAME:user.name} session:%{WORD:cisco_secure_email_gateway.log.session}"
+ - "^User %{USERNAME:user.name} %{GREEDYDATA:cisco_secure_email_gateway.log.action} of %{WORD:network.protocol} session %{IP:host.ip}"
+ - "^An authentication attempt by the user %{USERNAME:user.name} from %{IP:host.ip} %{WORD:cisco_secure_email_gateway.log.outcome} using an %{WORD:network.protocol} connection."
+ - "^The user %{USERNAME:user.name} %{WORD:cisco_secure_email_gateway.log.outcome} %{GREEDYDATA:cisco_secure_email_gateway.log.action} from %{IP:host.ip} with privilege %{DATA:cisco_secure_email_gateway.log.privilege} using an %{WORD:network.protocol} connection."
+ - "^User %{USERNAME:user.name} was %{WORD:cisco_secure_email_gateway.log.action} %{WORD:cisco_secure_email_gateway.log.outcome}."
+ - "^User %{USERNAME:user.name} %{WORD:cisco_secure_email_gateway.log.outcome} %{WORD:cisco_secure_email_gateway.log.action}"
+ - "^%{GREEDYDATA:cisco_secure_email_gateway.log.message}"
+ - lowercase:
+ field: network.protocol
+ ignore_failure: true
+ - set:
+ field: event.outcome
+ if: ctx?.cisco_secure_email_gateway?.log?.outcome == "failed"
+ value: failure 
+ - set:
+ field: event.outcome
+ if: ctx?.cisco_secure_email_gateway?.log?.outcome == "successfully"
+ value: success 
+ - set:
+ field: event.type
+ if: ctx?.cisco_secure_email_gateway?.log?.action == "logged on" || ctx?.cisco_secure_email_gateway?.log?.action == 'authenticated'
+ value: [start] 
+ - set:
+ field: event.type
+ if: ctx?.cisco_secure_email_gateway?.log?.action == 'logged out' || ctx?.cisco_secure_email_gateway?.log?.action == 'logout'
+ value: [end] 
+ - append:
+ field: related.user
+ value: "{{{user.name}}}"
+ if: ctx?.user?.name != null
+ allow_duplicates: false
+ ignore_failure: true
+ - append:
+ field: related.ip
+ value: "{{{host.ip}}}"
+ if: ctx?.host?.ip != null
+ allow_duplicates: false
+ ignore_failure: true

packages/cisco_secure_email_gateway/0.1.0/data_stream/log/elasticsearch/ingest_pipeline/pipeline_bounce.yml

@@ -0,0 +1,11 @@
+---
+processors:
+ - set:
+ field: event.kind
+ value: event
+ - grok:
+ field: cisco_secure_email_gateway.log.message
+ patterns:
+ - "^%{WORD:cisco_secure_email_gateway.log.bounce_type}: DCID %{NUMBER:cisco_secure_email_gateway.log.delivery_connection_id} MID %{NUMBER:email.message_id} From:<%{GREEDYDATA:email.from.address}> To:<%{GREEDYDATA:email.to.address}> RID %{NUMBER:cisco_secure_email_gateway.log.recipient_id} - %{DATA:cisco_secure_email_gateway.log.error_code} - %{GREEDYDATA:event.reason} \\(%{GREEDYDATA:cisco_secure_email_gateway.log.response}\\)"
+ - "^%{WORD:cisco_secure_email_gateway.log.bounce_type}: %{NUMBER:email.message_id}:%{NUMBER:cisco_secure_email_gateway.log.recipient_id} From:<%{GREEDYDATA:email.from.address}> To:<%{GREEDYDATA:email.to.address}>"
+ - "^%{GREEDYDATA:cisco_secure_email_gateway.log.message}"