module Fluent::PluginHelper::CompatParameters

Constants

BUFFER_PARAMS

This plugin helper is to bring old-fashioned buffer/other configuration parameters to v0.14 plugin API configurations. This helper is mainly to convert plugins from v0.12 API to v0.14 API safely, without breaking user deployment.

BUFFER_TIME_SLICED_PARAMS
EXTRACT_PARAMS
FORMATTER_PARAMS
INJECT_PARAMS
PARSER_PARAMS

Public Instance Methods

compat_parameters_buffer(conf, default_chunk_key: '') click to toggle source
# File lib/fluent/plugin_helper/compat_parameters.rb, line 133
def compat_parameters_buffer(conf, default_chunk_key: '')
  # return immediately if <buffer> section exists, or any buffer-related parameters don't exist
  return unless conf.elements('buffer').empty?
  return if (BUFFER_PARAMS.keys + BUFFER_TIME_SLICED_PARAMS.keys).all?{|k| !conf.has_key?(k) }

  # TODO: warn obsolete parameters if these are deprecated
  buffer_params = BUFFER_PARAMS.merge(BUFFER_TIME_SLICED_PARAMS)
  hash = compat_parameters_copy_to_subsection_attributes(conf, buffer_params) do |compat_key, value|
    if compat_key == 'buffer_queue_full_action' && value == 'exception'
      'throw_exception'
    else
      value
    end
  end

  chunk_key = default_chunk_key

  if conf.has_key?('time_slice_format')
    chunk_key = 'time'
    hash['timekey'] = case conf['time_slice_format']
                      when /\%S/ then 1
                      when /\%M/ then 60
                      when /\%H/ then 3600
                      when /\%d/ then 86400
                      else
                        raise Fluent::ConfigError, "time_slice_format only with %Y or %m is too long"
                      end
    if conf.has_key?('localtime') || conf.has_key?('utc')
      if conf.has_key?('localtime') && conf.has_key?('utc')
        raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
      elsif conf.has_key?('localtime')
        hash['timekey_use_utc'] = !(Fluent::Config.bool_value(conf['localtime']))
      elsif conf.has_key?('utc')
        hash['timekey_use_utc'] = Fluent::Config.bool_value(conf['utc'])
      end
    end
  else
    if chunk_key == 'time'
      hash['timekey'] = 86400 # TimeSliceOutput.time_slice_format default value is '%Y%m%d'
    end
  end

  e = Fluent::Config::Element.new('buffer', chunk_key, hash, [])
  conf.elements << e

  conf
end
compat_parameters_convert(conf, *types, **kwargs) click to toggle source
# File lib/fluent/plugin_helper/compat_parameters.rb, line 112
def compat_parameters_convert(conf, *types, **kwargs)
  types.each do |type|
    case type
    when :buffer
      compat_parameters_buffer(conf, **kwargs)
    when :inject
      compat_parameters_inject(conf)
    when :extract
      compat_parameters_extract(conf)
    when :parser
      compat_parameters_parser(conf)
    when :formatter
      compat_parameters_formatter(conf)
    else
      raise "BUG: unknown compat_parameters type: #{type}"
    end
  end

  conf
end
compat_parameters_copy_to_subsection_attributes(conf, params, &block) click to toggle source
# File lib/fluent/plugin_helper/compat_parameters.rb, line 327
def compat_parameters_copy_to_subsection_attributes(conf, params, &block)
  hash = {}
  params.each do |compat, current|
    next unless current
    if conf.has_key?(compat)
      if block_given?
        hash[current] = block.call(compat, conf[compat])
      else
        hash[current] = conf[compat]
      end
    end
  end
  hash
end
compat_parameters_extract(conf) click to toggle source
# File lib/fluent/plugin_helper/compat_parameters.rb, line 230
def compat_parameters_extract(conf)
  return unless conf.elements('extract').empty?
  return if EXTRACT_PARAMS.keys.all?{|k| !conf.has_key?(k) } && !conf.has_key?('format')

  # TODO: warn obsolete parameters if these are deprecated
  hash = compat_parameters_copy_to_subsection_attributes(conf, EXTRACT_PARAMS)

  if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
    hash['time_key'] ||= 'time'
    hash['time_type'] = 'unixtime'
  elsif conf.has_key?('format') && conf["format"].start_with?("/") && conf["format"].end_with?("/") # old-style regexp parser
    hash['time_key'] ||= 'time'
    hash['time_type'] ||= 'string'
  end
  if conf.has_key?('localtime') || conf.has_key?('utc')
    if conf.has_key?('localtime') && conf.has_key?('utc')
      raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
    elsif conf.has_key?('localtime')
      hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
    elsif conf.has_key?('utc')
      hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
      # Specifying "localtime false" means using UTC in TimeFormatter
      # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
      # There are difference between "Z" and "+0000" in timezone formatting.
      # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
    end
  end

  e = Fluent::Config::Element.new('extract', '', hash, [])
  conf.elements << e

  conf
end
compat_parameters_formatter(conf) click to toggle source
# File lib/fluent/plugin_helper/compat_parameters.rb, line 310
def compat_parameters_formatter(conf)
  return unless conf.elements('format').empty?
  return if FORMATTER_PARAMS.keys.all?{|k| !conf.has_key?(k) }

  # TODO: warn obsolete parameters if these are deprecated
  hash = compat_parameters_copy_to_subsection_attributes(conf, FORMATTER_PARAMS)

  if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
    hash['time_type'] = 'unixtime'
  end

  e = Fluent::Config::Element.new('format', '', hash, [])
  conf.elements << e

  conf
end
compat_parameters_inject(conf) click to toggle source
# File lib/fluent/plugin_helper/compat_parameters.rb, line 181
def compat_parameters_inject(conf)
  return unless conf.elements('inject').empty?
  return if INJECT_PARAMS.keys.all?{|k| !conf.has_key?(k) }

  # TODO: warn obsolete parameters if these are deprecated
  hash = compat_parameters_copy_to_subsection_attributes(conf, INJECT_PARAMS)

  if conf.has_key?('include_time_key') && Fluent::Config.bool_value(conf['include_time_key'])
    hash['time_key'] ||= 'time'
    hash['time_type'] ||= 'string'
  end
  if conf.has_key?('time_as_epoch') && Fluent::Config.bool_value(conf['time_as_epoch'])
    hash['time_key'] ||= 'time'
    hash['time_type'] = 'unixtime'
  end
  if conf.has_key?('localtime') || conf.has_key?('utc')
    utc = to_bool(conf['utc'])
    localtime = to_bool(conf['localtime'])
    if conf.has_key?('localtime') && conf.has_key?('utc') && !(localtime ^ utc)
      raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
    elsif conf.has_key?('localtime')
      hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
    elsif conf.has_key?('utc')
      hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
      # Specifying "localtime false" means using UTC in TimeFormatter
      # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
      # There are difference between "Z" and "+0000" in timezone formatting.
      # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
    end
  end

  if conf.has_key?('include_tag_key') && Fluent::Config.bool_value(conf['include_tag_key'])
    hash['tag_key'] ||= 'tag'
  end

  e = Fluent::Config::Element.new('inject', '', hash, [])
  conf.elements << e

  conf
end
compat_parameters_parser(conf) click to toggle source
# File lib/fluent/plugin_helper/compat_parameters.rb, line 264
def compat_parameters_parser(conf)
  return unless conf.elements('parse').empty?
  return if PARSER_PARAMS.keys.all?{|k| !conf.has_key?(k) }

  # TODO: warn obsolete parameters if these are deprecated
  hash = compat_parameters_copy_to_subsection_attributes(conf, PARSER_PARAMS)

  if conf["format"]
    if conf["format"].start_with?("/") && conf["format"].end_with?("/")
      hash["@type"] = "regexp"
      hash["expression"] = conf["format"][1..-2]
    else
      hash["@type"] = conf["format"]
    end
  end

  if conf["types"]
    delimiter = conf["types_delimiter"] || ','
    label_delimiter = conf["types_label_delimiter"] || ':'
    types = {}
    conf['types'].split(delimiter).each do |pair|
      key, value = pair.split(label_delimiter, 2)
      types[key] = value
    end
    hash["types"] = JSON.dump(types)
  end
  if conf.has_key?('localtime') || conf.has_key?('utc')
    if conf.has_key?('localtime') && conf.has_key?('utc')
      raise Fluent::ConfigError, "both of utc and localtime are specified, use only one of them"
    elsif conf.has_key?('localtime')
      hash['localtime'] = Fluent::Config.bool_value(conf['localtime'])
    elsif conf.has_key?('utc')
      hash['localtime'] = !(Fluent::Config.bool_value(conf['utc']))
      # Specifying "localtime false" means using UTC in TimeFormatter
      # And specifying "utc" is different from specifying "timezone +0000"(it's not always UTC).
      # There are difference between "Z" and "+0000" in timezone formatting.
      # TODO: add kwargs to TimeFormatter to specify "using localtime", "using UTC" or "using specified timezone" in more explicit way
    end
  end

  e = Fluent::Config::Element.new('parse', '', hash, [])
  conf.elements << e

  conf
end
to_bool(v) click to toggle source
# File lib/fluent/plugin_helper/compat_parameters.rb, line 222
def to_bool(v)
  if  v.is_a?(FalseClass) || v == 'false' || v.nil?
    false
  else
    true
  end
end