HOME

sh-3ll 1.0
DIR:/proc/self/root/usr/share/filebeat/module/threatintel/misp/ingest/
Current File : //proc/self/root/usr/share/filebeat/module/threatintel/misp/ingest/pipeline.yml
---
description: Pipeline for parsing MISP Threat Intel
processors:
  ####################
  # Event ECS fields #
  ####################
  - set:
      field: event.ingested
      value: "{{_ingest.timestamp}}"
  - set:
      field: event.kind
      value: enrichment
  - set:
      field: event.category
      value: threat
  - set:
      field: event.type
      value: indicator

  ######################
  # General ECS fields #
  ######################
  - rename:
      field: message
      target_field: event.original
      ignore_missing: true
  - json:
      field: event.original
      target_field: json
  - fingerprint:
      fields:
        - json.Event.Attribute.uuid
        - json.Event.Object.Attribute.uuid
      target_field: "_id"
      ignore_missing: true
  - rename:
      field: json.Event
      target_field: misp
      ignore_missing: true
  - set:
      field: threat.indicator.provider
      value: misp
      if: ctx.misp?.Orgc?.local != 'false'
  - set:
      field: threat.indicator.provider
      value: "{{misp.Orgc.name}}"
      if: ctx.misp?.Orgc?.local == 'false'
      ignore_empty_value: true

  # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event
  - remove:
      field:
        - misp.ShadowAttribute
        - misp.RelatedEvent
        - misp.Galaxy
        - misp.Attribute.Galaxy
        - misp.Attribute.ShadowAttribute
        - misp.EventReport
        - misp.Object.Attribute.Galaxy
        - misp.Object.Attribute.ShadowAttribute
      ignore_missing: true
  - remove:
      field:
        - misp.Attribute
      ignore_missing: true
      if: ctx.misp?.Attribute.size() == 0
  - remove:
      field:
        - misp.Object
      ignore_missing: true
      if: ctx.misp?.Object.size() == 0
  - date:
      field: misp.timestamp
      formats:
        - UNIX
      ignore_failure: true
  - rename:
      field: misp.Attribute
      target_field: misp.attribute
      ignore_missing: true
  - rename:
      field: misp.Object
      target_field: misp.object
      ignore_missing: true
  - rename:
      field: misp.object.Attribute
      target_field: misp.object.attribute
      ignore_missing: true
  - rename:
      field: misp.Orgc
      target_field: misp.orgc
      ignore_missing: true
  - rename:
      field: misp.Org
      target_field: misp.org
      ignore_missing: true
  - rename:
      field: misp.Tag
      target_field: misp.tag
      ignore_missing: true

  # # Dance around issue of not being able to split the document into two.
  # # Make the Object.Attribute field primary if it exists, but keep the
  # # outer Attribute as context.
  - rename:
      field: misp.attribute
      target_field: misp.context.attribute
      ignore_missing: true
      if: ctx.misp?.object != null
  - rename:
      field: misp.object.attribute
      target_field: misp.attribute
      ignore_missing: true
      if: ctx.misp?.object != null

  #####################
  # Threat ECS Fields #
  #####################
  - set:
      field: threat.feed.name
      value: "[Filebeat] MISP"
  - set:
      field: threat.feed.dashboard_id
      value: "ad9c7430-72de-11eb-a3e3-b3cc7c78a70f"
  - rename:
      field: misp.attribute.first_seen
      target_field: threat.indicator.first_seen
      ignore_missing: true
  - rename:
      field: misp.attribute.last_seen
      target_field: threat.indicator.last_seen
      ignore_missing: true
  - convert:
      field: misp.analysis
      type: long
      target_field: threat.indicator.scanner_stats
      ignore_missing: true
  - convert:
      field: misp.threat_level_id
      type: long
      ignore_missing: true

  ## File/Hash indicator operations
  - set:
      field: threat.indicator.type
      value: file
      if: "ctx.misp?.attribute?.type != null && (['md5', 'impfuzzy', 'imphash', 'pehash', 'sha1', 'sha224', 'sha256', 'sha3-224', 'sha3-256', 'sha3-384', 'sha3-512', 'sha384', 'sha512', 'sha512/224', 'sha512/256', 'ssdeep', 'tlsh', 'vhash'].contains(ctx.misp?.attribute?.type) || ctx.misp?.attribute?.type.startsWith('filename'))"
  - rename:
      field: misp.attribute.value
      target_field: "threat.indicator.file.hash.{{misp.attribute.type}}"
      ignore_missing: true
      if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type != null && !ctx.misp?.attribute?.type.startsWith('filename')"
  - rename:
      field: misp.attribute.value
      target_field: threat.indicator.file.name
      ignore_missing: true
      if: "ctx.threat?.indicator?.type == 'file' && ctx.misp?.attribute?.type == 'filename'"
  - grok:
      field: misp.attribute.type
      patterns:
        - "%{WORD}\\|%{WORD:_tmp.hashtype}"
      ignore_missing: true
      if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
  - grok:
      field: misp.attribute.value
      patterns:
        - "%{DATA:threat.indicator.file.name}\\|%{GREEDYDATA:_tmp.hashvalue}"
      ignore_missing: true
      if: ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|')
  - set:
      field: threat.indicator.file.hash.{{_tmp.hashtype}}
      value: "{{_tmp.hashvalue}}"
      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('filename|') && ctx?._tmp?.hashvalue != null && ctx?._tmp?.hashtype != null"

  ## URL/URI indicator operations
  - set:
      field: threat.indicator.type
      value: url
      if: "ctx.misp?.attribute?.type != null && ['url', 'link', 'uri'].contains(ctx.misp?.attribute?.type)"
  - uri_parts:
      field: misp.attribute.value
      target_field: threat.indicator.url
      keep_original: true
      remove_if_successful: true
      if: ctx.threat?.indicator?.type == 'url' && ctx.misp?.attribute?.type != 'uri'
  - set:
      field: threat.indicator.url.full
      value: "{{{threat.indicator.url.original}}}"
      ignore_empty_value: true
      if: "ctx.threat?.indicator?.type == 'url' && ctx.misp?.attribute?.type != 'uri'"

  ## Regkey indicator operations
  - set:
      field: threat.indicator.type
      value: windows-registry-key
      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('regkey')"
  - rename:
      field: misp.attribute.value
      target_field: threat.indicator.registry.key
      ignore_missing: true
      if: "ctx.threat?.indicator?.type == 'windows-registry-key' && ctx.misp?.attribute?.type == 'regkey'"
  - grok:
      field: misp.attribute.value
      patterns:
        - "%{DATA:threat.indicator.registry.key}\\|%{DATA:threat.indicator.registry.value}"
      ignore_missing: true
      if: "ctx.misp?.attribute?.type == 'regkey|value'"

  ## AS indicator operations
  - set:
      field: threat.indicator.type
      value: autonomous-system
      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type == 'AS'"
  - convert:
      field: misp.attribute.value
      type: long
      target_field: threat.indicator.as.number
      ignore_missing: true
      if: ctx.threat?.indicator?.type == 'autonomous-system'

  ## Domain/IP/Port indicator operations
  - set:
      field: threat.indicator.type
      value: domain-name
      if: "ctx.misp?.attribute?.type != null && (ctx.misp?.attribute?.type == 'hostname' || ctx.misp?.attribute?.type.startsWith('domain'))"
  - set:
      field: threat.indicator.type
      value: ipv4-addr
      if: "ctx.misp?.attribute?.type != null && ['ip-src', 'ip-src|port', 'ip-dst', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
  - rename:
      field: misp.attribute.value
      target_field: threat.indicator.url.domain
      ignore_missing: true
      if: "ctx.threat?.indicator?.type == 'domain-name' && ctx.misp?.attribute?.type != 'domain|ip' && ctx.threat?.indicator?.url?.domain == null"
  - rename:
      field: misp.attribute.value
      target_field: threat.indicator.ip
      ignore_missing: true
      if: "ctx.threat?.indicator?.type == 'ipv4-addr' && ctx.misp?.attribute?.type != 'domain|ip' && !['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"
  - grok:
      field: misp.attribute.value
      patterns:
        - "%{DATA:threat.indicator.url.domain}\\|%{IP:threat.indicator.ip}"
      ignore_missing: true
      if: ctx.misp?.attribute?.type == 'domain|ip' && ctx.threat?.indicator?.url?.domain == null
  - grok:
      field: misp.attribute.value
      patterns:
        - "%{IP:threat.indicator.ip}\\|%{NUMBER:threat.indicator.port}"
      ignore_missing: true
      if: "['ip-src|port', 'ip-dst|port'].contains(ctx.misp?.attribute?.type)"

  ## Email indicator operations
  # Currently this ignores email-message, except setting the type it will leave the rest of the fields under misp.
  - set:
      field: threat.indicator.type
      value: email-addr
      if: "ctx.misp?.attribute?.type != null && ['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
  - set:
      field: threat.indicator.type
      value: email-message
      if: "ctx.misp?.attribute?.type != null && ctx.misp?.attribute?.type.startsWith('email') && !['email-dst', 'email-src'].contains(ctx.misp?.attribute?.type)"
  - rename:
      field: misp.attribute.value
      target_field: threat.indicator.email.address
      ignore_missing: true
      if: ctx.threat?.indicator?.type == 'email-addr'
  - rename:
      field: misp.event_creator_email
      target_field: user.email
      ignore_missing: true
  - append:
      field: user.roles
      value: "reporting_user"
      if: ctx?.user?.email != null

  ## MAC Address indicator operations
  - set:
      field: threat.indicator.type
      value: mac-addr
      if: "ctx.misp?.attribute?.type != null && ['mac-address', 'mac-eui-64'].contains(ctx.misp?.attribute?.type)"
  - rename:
      field: misp.attribute.value
      target_field: threat.indicator.mac
      ignore_missing: true
      if: ctx.threat?.indicator?.type == 'mac-addr'

  ###################
  # Tags ECS fields #
  ###################
  # Stripping special characters from tags
  - script:
      lang: painless
      if: ctx.misp?.tag != null
      source: |
        def tags = ctx.misp.tag.stream()
           .map(t -> t.name.replace('\\', '').replace('"', ''))
           .collect(Collectors.toList());
        def tlpTags = tags.stream()
           .filter(t -> t.startsWith('tlp:'))
           .map(t -> t.replace('tlp:', ''))
           .collect(Collectors.toList());

        ctx.tags = tags;
        ctx.threat.indicator.marking = [ 'tlp': tlpTags ];

  # Setting indicator type to unknown if it does not match anything
  - set:
      field: threat.indicator.type
      value: unknown
      if: ctx.threat?.indicator?.type == null

  #################
  # Convert types #
  #################
  - convert:
      field: misp.attribute.distribution
      type: long
      ignore_missing: true
  - convert:
      field: misp.context.attribute.distribution
      type: long
      ignore_missing: true
  - convert:
      field: threat.indicator.port
      type: long
      ignore_missing: true
  - convert:
      field: misp.attribute_count
      type: long
      ignore_missing: true

  ######################
  # Cleanup processors #
  ######################
  - remove:
      field: event.original
      if: "ctx?.tags == null || !(ctx.tags.contains('preserve_original_event'))"
      ignore_failure: true
      ignore_missing: true
  - script:
      lang: painless
      if: ctx?.misp != null
      source: |
        void handleMap(Map map) {
          for (def x : map.values()) {
            if (x instanceof Map) {
                handleMap(x);
            } else if (x instanceof List) {
                handleList(x);
            }
          }
        map.values().removeIf(v -> v == null);
        }
        void handleList(List list) {
          for (def x : list) {
              if (x instanceof Map) {
                  handleMap(x);
              } else if (x instanceof List) {
                  handleList(x);
              }
          }
        }
        handleMap(ctx);
  # Removing fields not needed anymore, either because its copied somewhere else, or is not relevant to this event
  - remove:
      field:
        - misp.attribute.value
      ignore_missing: true
      if: ctx.threat?.indicator?.type != 'unknown'
  - remove:
      field:
        # This removes a number of fields that may be wanted in the future when
        # misp.attribute and misp.object.attribute can
        # be separated. At the root of .object are fields that mirror fields at
        # the root of misp.
        - misp.object
      ignore_missing: true
  - remove:
      field:
        - misp.Attribute.timestamp
        - misp.timestamp
        - misp.tag
        - misp.org
        - misp.analysis
        - _tmp
        - json
      ignore_missing: true

on_failure:
  - set:
      field: error.message
      value: "{{ _ingest.on_failure_message }}"