class Asciidoctor::PreprocessorReader

Public: Methods for retrieving lines from AsciiDoc source files, evaluating preprocessor directives as each line is read off the Array of lines.

Attributes

include_stack[R]
includes[R]

Public Class Methods

new(document, data = nil, cursor = nil, opts = {}) click to toggle source

Public: Initialize the PreprocessorReader object

Calls superclass method Asciidoctor::Reader.new
# File lib/asciidoctor/reader.rb, line 566
def initialize document, data = nil, cursor = nil, opts = {}
  @document = document
  super data, cursor, opts
  include_depth_default = document.attributes.fetch('max-include-depth', 64).to_i
  include_depth_default = 0 if include_depth_default < 0
  # track both absolute depth for comparing to size of include stack and relative depth for reporting
  @maxdepth = {:abs => include_depth_default, :rel => include_depth_default}
  @include_stack = []
  @includes = document.catalog[:includes]
  @skipping = false
  @conditional_stack = []
  @include_processor_extensions = nil
end

Public Instance Methods

empty?() click to toggle source

(see Asciidoctor::Reader#empty?)

# File lib/asciidoctor/reader.rb, line 669
def empty?
  peek_line ? false : true
end
Also aliased as: eof?
eof?()
Alias for: empty?
exceeded_max_depth?() click to toggle source
# File lib/asciidoctor/reader.rb, line 1116
def exceeded_max_depth?
  if (abs_maxdepth = @maxdepth[:abs]) > 0 && @include_stack.size >= abs_maxdepth
    @maxdepth[:rel]
  else
    false
  end
end
has_more_lines?() click to toggle source

(see Asciidoctor::Reader#has_more_lines?)

# File lib/asciidoctor/reader.rb, line 664
def has_more_lines?
  peek_line ? true : false
end
include_depth() click to toggle source
# File lib/asciidoctor/reader.rb, line 1112
def include_depth
  @include_stack.size
end
include_processors?() click to toggle source
# File lib/asciidoctor/reader.rb, line 1226
def include_processors?
  if @include_processor_extensions.nil?
    if @document.extensions? && @document.extensions.include_processors?
      !!(@include_processor_extensions = @document.extensions.include_processors)
    else
      @include_processor_extensions = false
    end
  else
    @include_processor_extensions != false
  end
end
peek_line(direct = false) click to toggle source

Public: Override the Asciidoctor::Reader#peek_line method to pop the include stack if the last line has been reached and there's at least one include on the stack.

Returns the next line of the source data as a String if there are lines remaining in the current include context or a parent include context. Returns nothing if there are no more lines remaining and the include stack is empty.

Calls superclass method Asciidoctor::Reader#peek_line
# File lib/asciidoctor/reader.rb, line 681
def peek_line direct = false
  if (line = super)
    line
  elsif @include_stack.empty?
    nil
  else
    pop_include
    peek_line direct
  end
end
pop_include() click to toggle source
# File lib/asciidoctor/reader.rb, line 1101
def pop_include
  if @include_stack.size > 0
    @lines, @file, @dir, @path, @lineno, @maxdepth, @process_lines = @include_stack.pop
    # FIXME kind of a hack
    #Document::AttributeEntry.new('infile', @file).save_to_next_block @document
    #Document::AttributeEntry.new('indir', ::File.dirname(@file)).save_to_next_block @document
    @look_ahead = 0
    nil
  end
end
prepare_lines(data, opts = {}) click to toggle source
Calls superclass method Asciidoctor::Reader#prepare_lines
# File lib/asciidoctor/reader.rb, line 580
def prepare_lines data, opts = {}
  result = super

  # QUESTION should this work for AsciiDoc table cell content? Currently it does not.
  if @document && (@document.attributes.key? 'skip-front-matter')
    if (front_matter = skip_front_matter! result)
      @document.attributes['front-matter'] = front_matter * LF
    end
  end

  if opts.fetch :condense, true
    result.shift && @lineno += 1 while (first = result[0]) && first.empty?
    result.pop while (last = result[-1]) && last.empty?
  end

  if opts[:indent]
    Parser.adjust_indentation! result, opts[:indent], (@document.attr 'tabsize')
  end

  result
end
preprocess_conditional_directive(keyword, target, delimiter, text) click to toggle source

Internal: Preprocess the directive to conditionally include or exclude content.

Preprocess the conditional directive (ifdef, ifndef, ifeval, endif) under the cursor. If Reader is currently skipping content, then simply track the open and close delimiters of any nested conditional blocks. If Reader is not skipping, mark whether the condition is satisfied and continue preprocessing recursively until the next line of available content is found.

keyword - The conditional inclusion directive (ifdef, ifndef, ifeval, endif) target - The target, which is the name of one or more attributes that are

used in the condition (blank in the case of the ifeval directive)

delimiter - The conditional delimiter for multiple attributes ('+' means all

attributes must be defined or undefined, ',' means any of the attributes
can be defined or undefined.

text - The text associated with this directive (occurring between the square brackets)

Used for a single-line conditional block in the case of the ifdef or
ifndef directives, and for the conditional expression for the ifeval directive.

Returns a Boolean indicating whether the cursor should be advanced

# File lib/asciidoctor/reader.rb, line 712
def preprocess_conditional_directive keyword, target, delimiter, text
  # attributes are case insensitive
  target = target.downcase unless (no_target = target.empty?)

  # must have a target before brackets if ifdef or ifndef
  # must not have text between brackets if endif
  # skip line if it doesn't meet this criteria
  # QUESTION should we warn for these bogus declarations?
  return false if (no_target && (keyword == 'ifdef' || keyword == 'ifndef')) || (text && keyword == 'endif')

  if keyword == 'endif'
    if @conditional_stack.empty?
      warn %(asciidoctor: ERROR: #{line_info}: unmatched macro: endif::#{target}[])
    elsif no_target || target == (pair = @conditional_stack[-1])[:target]
      @conditional_stack.pop
      @skipping = @conditional_stack.empty? ? false : @conditional_stack[-1][:skipping]
    else
      warn %(asciidoctor: ERROR: #{line_info}: mismatched macro: endif::#{target}[], expected endif::#{pair[:target]}[])
    end
    return true
  end

  if @skipping
    skip = false
  else
    # QUESTION any way to wrap ifdef & ifndef logic up together?
    case keyword
    when 'ifdef'
      case delimiter
      when ','
        # skip if no attribute is defined
        skip = target.split(',', -1).none? {|name| @document.attributes.key? name }
      when '+'
        # skip if any attribute is undefined
        skip = target.split('+', -1).any? {|name| !@document.attributes.key? name }
      else
        # if the attribute is undefined, then skip
        skip = !@document.attributes.key?(target)
      end
    when 'ifndef'
      case delimiter
      when ','
        # skip if any attribute is defined
        skip = target.split(',', -1).any? {|name| @document.attributes.key? name }
      when '+'
        # skip if all attributes are defined
        skip = target.split('+', -1).all? {|name| @document.attributes.key? name }
      else
        # if the attribute is defined, then skip
        skip = @document.attributes.key?(target)
      end
    when 'ifeval'
      # the text in brackets must match an expression
      # don't honor match if it doesn't meet this criteria
      return false unless no_target && EvalExpressionRx =~ text.strip

      # NOTE save values eagerly for Ruby 1.8.7 compat
      lhs, op, rhs = $1, $2, $3
      lhs = resolve_expr_val lhs
      rhs = resolve_expr_val rhs

      # regex enforces a restricted set of math-related operations
      if op == '!='
        skip = lhs.send :==, rhs
      else
        skip = !(lhs.send op.to_sym, rhs)
      end
    end
  end

  # conditional inclusion block
  if keyword == 'ifeval' || !text
    @skipping = true if skip
    @conditional_stack << {:target => target, :skip => skip, :skipping => @skipping}
  # single line conditional inclusion
  else
    unless @skipping || skip
      replace_next_line text.rstrip
      # HACK push dummy line to stand in for the opening conditional directive that's subsequently dropped
      unshift ''
      # NOTE force line to be processed again if it looks like an include directive
      # QUESTION should we just call preprocess_include_directive here?
      @look_ahead -= 1 if text.start_with? 'include::'
    end
  end

  true
end
preprocess_include_directive(raw_target, raw_attributes) click to toggle source

Internal: Preprocess the directive to include lines from another document.

Preprocess the directive to include the target document. The scenarios are as follows:

If SafeMode is SECURE or greater, the directive is ignore and the include directive line is emitted verbatim.

Otherwise, if an include processor is specified pass the target and attributes to that processor and expect an Array of String lines in return.

Otherwise, if the max depth is greater than 0, and is not exceeded by the stack size, normalize the target path and read the lines onto the beginning of the Array of source data.

If none of the above apply, emit the include directive line verbatim.

target - The name of the source document to include as specified in the

target slot of the include::[] directive

Returns a Boolean indicating whether the line under the cursor has changed.

# File lib/asciidoctor/reader.rb, line 822
def preprocess_include_directive raw_target, raw_attributes
  if ((target = raw_target).include? ATTR_REF_HEAD) &&
      (target = @document.sub_attributes raw_target, :attribute_missing => 'drop-line').empty?
    shift
    if @document.attributes.fetch('attribute-missing', Compliance.attribute_missing) == 'skip'
      unshift %(Unresolved directive in #{@path} - include::#{raw_target}[#{raw_attributes}])
    end
    true
  elsif include_processors? && (ext = @include_processor_extensions.find {|candidate| candidate.instance.handles? target })
    shift
    # FIXME parse attributes only if requested by extension
    ext.process_method[@document, self, target, AttributeList.new(raw_attributes).parse]
    true
  # if running in SafeMode::SECURE or greater, don't process this directive
  # however, be friendly and at least make it a link to the source document
  elsif @document.safe >= SafeMode::SECURE
    # FIXME we don't want to use a link macro if we are in a verbatim context
    replace_next_line %(link:#{target}[])
    true
  elsif (abs_maxdepth = @maxdepth[:abs]) > 0
    if @include_stack.size >= abs_maxdepth
      warn %(asciidoctor: ERROR: #{line_info}: maximum include depth of #{@maxdepth[:rel]} exceeded)
      return false
    end
    if ::RUBY_ENGINE_OPAL && ::JAVASCRIPT_IO_MODULE == 'xmlhttprequest'
      # NOTE resolves uri relative to currently loaded document
      # NOTE we defer checking if file exists and catch the 404 error if it does not
      target_type = :file
      inc_path = relpath = @include_stack.empty? && ::Dir.pwd == @document.base_dir ? target : (::File.join @dir, target)
    elsif Helpers.uriish? target
      unless @document.attributes.key? 'allow-uri-read'
        replace_next_line %(link:#{target}[])
        return true
      end

      target_type = :uri
      inc_path = relpath = target
      if @document.attributes.key? 'cache-uri'
        # caching requires the open-uri-cached gem to be installed
        # processing will be automatically aborted if these libraries can't be opened
        Helpers.require_library 'open-uri/cached', 'open-uri-cached' unless defined? ::OpenURI::Cache
      elsif !::RUBY_ENGINE_OPAL
        # autoload open-uri
        ::OpenURI
      end
    else
      target_type = :file
      # include file is resolved relative to dir of current include, or base_dir if within original docfile
      inc_path = @document.normalize_system_path target, @dir, nil, :target_name => 'include file'
      unless ::File.file? inc_path
        warn %(asciidoctor: WARNING: #{line_info}: include file not found: #{inc_path})
        replace_next_line %(Unresolved directive in #{@path} - include::#{target}[#{raw_attributes}])
        return true
      end
      # NOTE relpath is the path relative to the root document (or base_dir, if set)
      # QUESTION should we move relative_path method to Document
      relpath = (@path_resolver ||= PathResolver.new).relative_path inc_path, @document.base_dir
    end

    inc_linenos, inc_tags, attributes = nil, nil, {}
    unless raw_attributes.empty?
      # QUESTION should we use @document.parse_attribues?
      attributes = AttributeList.new(raw_attributes).parse
      if attributes.key? 'lines'
        inc_linenos = []
        attributes['lines'].split(DataDelimiterRx).each do |linedef|
          if linedef.include?('..')
            from, to = linedef.split('..', 2).map {|it| it.to_i }
            if to == -1
              inc_linenos << from
              inc_linenos << 1.0/0.0
            else
              inc_linenos.concat ::Range.new(from, to).to_a
            end
          else
            inc_linenos << linedef.to_i
          end
        end
        inc_linenos = inc_linenos.empty? ? nil : inc_linenos.sort.uniq
      elsif attributes.key? 'tag'
        unless (tag = attributes['tag']).empty?
          if tag.start_with? '!'
            inc_tags = { (tag.slice 1, tag.length) => false } unless tag == '!'
          else
            inc_tags = { tag => true }
          end
        end
      elsif attributes.key? 'tags'
        inc_tags = {}
        attributes['tags'].split(DataDelimiterRx).each do |tagdef|
          if tagdef.start_with? '!'
            inc_tags[tagdef.slice 1, tagdef.length] = false unless tagdef == '!'
          else
            inc_tags[tagdef] = true
          end unless tagdef.empty?
        end
        inc_tags = nil if inc_tags.empty?
      end
    end

    if inc_linenos
      inc_lines, inc_offset, inc_lineno = [], nil, 0
      begin
        open(inc_path, 'r') do |f|
          f.each_line do |l|
            inc_lineno += 1
            select = inc_linenos[0]
            if ::Float === select && select.infinite?
              # NOTE record line where we started selecting
              inc_offset ||= inc_lineno
              inc_lines << l
            else
              if inc_lineno == select
                # NOTE record line where we started selecting
                inc_offset ||= inc_lineno
                inc_lines << l
                inc_linenos.shift
              end
              break if inc_linenos.empty?
            end
          end
        end
      rescue
        warn %(asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{inc_path})
        replace_next_line %(Unresolved directive in #{@path} - include::#{target}[#{raw_attributes}])
        return true
      end
      shift
      # FIXME not accounting for skipped lines in reader line numbering
      push_include inc_lines, inc_path, relpath, inc_offset, attributes if inc_offset
    elsif inc_tags
      inc_lines, inc_offset, inc_lineno, tag_stack, tags_used, active_tag = [], nil, 0, [], ::Set.new, nil
      if inc_tags.key? '**'
        if inc_tags.key? '*'
          select = base_select = (inc_tags.delete '**')
          wildcard = inc_tags.delete '*'
        else
          select = base_select = wildcard = (inc_tags.delete '**')
        end
      else
        select = base_select = !(inc_tags.value? true)
        wildcard = inc_tags.delete '*'
      end
      if (ext_idx = inc_path.rindex '.') && (circ_cmt = CIRCUMFIX_COMMENTS[inc_path.slice ext_idx, inc_path.length])
        cmt_suffix_len = (tag_suffix = %([] #{circ_cmt[:suffix]})).length - 2
      end
      begin
        open(inc_path, 'r') do |f|
          f.each_line do |l|
            inc_lineno += 1
            # must force encoding since we're performing String operations on line
            l.force_encoding ::Encoding::UTF_8 if FORCE_ENCODING
            if (((tl = l.chomp).end_with? '[]') ||
                (tag_suffix && (tl.end_with? tag_suffix) && (tl = tl.slice 0, tl.length - cmt_suffix_len))) &&
                TagDirectiveRx =~ tl
              if $1 # end tag
                if (this_tag = $2) == active_tag
                  tag_stack.pop
                  active_tag, select = tag_stack.empty? ? [nil, base_select] : tag_stack[-1]
                elsif inc_tags.key? this_tag
                  if (idx = tag_stack.rindex {|key, _| key == this_tag })
                    idx == 0 ? tag_stack.shift : (tag_stack.delete_at idx)
                    warn %(asciidoctor: WARNING: #{target}: line #{inc_lineno}: mismatched end tag in include: expected #{active_tag}, found #{this_tag})
                  else
                    warn %(asciidoctor: WARNING: #{target}: line #{inc_lineno}: unexpected end tag in include: #{this_tag})
                  end
                end
              elsif inc_tags.key?(this_tag = $2)
                tags_used << this_tag
                # QUESTION should we prevent tag from being selected when enclosing tag is excluded?
                tag_stack << [(active_tag = this_tag), (select = inc_tags[this_tag])]
              elsif !wildcard.nil?
                select = active_tag && !select ? false : wildcard
                tag_stack << [(active_tag = this_tag), select]
              end
            elsif select
              # NOTE record the line where we started selecting
              inc_offset ||= inc_lineno
              inc_lines << l
            end
          end
        end
      rescue
        warn %(asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{inc_path})
        replace_next_line %(Unresolved directive in #{@path} - include::#{target}[#{raw_attributes}])
        return true
      end
      unless (missing_tags = inc_tags.keys.to_a - tags_used.to_a).empty?
        warn %(asciidoctor: WARNING: #{line_info}: tag#{missing_tags.size > 1 ? 's' : nil} '#{missing_tags * ','}' not found in include #{target_type}: #{inc_path})
      end
      shift
      # FIXME not accounting for skipped lines in reader line numbering
      push_include inc_lines, inc_path, relpath, inc_offset, attributes if inc_offset
    else
      begin
        # NOTE read content first so that we only advance cursor if IO operation succeeds
        inc_content = target_type == :file ? (::IO.read inc_path) : open(inc_path, 'r') {|f| f.read }
        shift
        push_include inc_content, inc_path, relpath, 1, attributes
      rescue
        warn %(asciidoctor: WARNING: #{line_info}: include #{target_type} not readable: #{inc_path})
        replace_next_line %(Unresolved directive in #{@path} - include::#{target}[#{raw_attributes}])
        return true
      end
    end
    true
  else
    false
  end
end
process_line(line) click to toggle source
# File lib/asciidoctor/reader.rb, line 602
def process_line line
  return line unless @process_lines

  if line.empty?
    @look_ahead += 1
    return line
  end

  # NOTE highly optimized
  if line.end_with?(']') && !line.start_with?('[') && line.include?('::')
    if (line.include? 'if') && ConditionalDirectiveRx =~ line
      # if escaped, mark as processed and return line unescaped
      if $1 == '\\'
        @unescape_next_line = true
        @look_ahead += 1
        line[1..-1]
      elsif preprocess_conditional_directive $2, $3, $4, $5
        # move the pointer past the conditional line
        shift
        # treat next line as uncharted territory
        nil
      else
        # the line was not a valid conditional line
        # mark it as visited and return it
        @look_ahead += 1
        line
      end
    elsif @skipping
      shift
      nil
    elsif (line.start_with? 'inc', '\\inc') && IncludeDirectiveRx =~ line
      # if escaped, mark as processed and return line unescaped
      if $1 == '\\'
        @unescape_next_line = true
        @look_ahead += 1
        line[1..-1]
      # QUESTION should we strip whitespace from raw attributes in Substitutors#parse_attributes? (check perf)
      elsif preprocess_include_directive $2, $3.strip
        # peek again since the content has changed
        nil
      else
        # the line was not a valid include line and is unchanged
        # mark it as visited and return it
        @look_ahead += 1
        line
      end
    else
      # NOTE optimization to inline super
      @look_ahead += 1
      line
    end
  elsif @skipping
    shift
    nil
  else
    # NOTE optimization to inline super
    @look_ahead += 1
    line
  end
end
push_include(data, file = nil, path = nil, lineno = 1, attributes = {}) click to toggle source

Public: Push source onto the front of the reader and switch the context based on the file, document-relative path and line information given.

This method is typically used in an IncludeProcessor to add source read from the target specified.

Examples

path = 'partial.adoc'
file = File.expand_path path
data = IO.read file
reader.push_include data, file, path

Returns this Reader object.

# File lib/asciidoctor/reader.rb, line 1047
def push_include data, file = nil, path = nil, lineno = 1, attributes = {}
  @include_stack << [@lines, @file, @dir, @path, @lineno, @maxdepth, @process_lines]
  if file
    @file = file
    @dir = File.dirname file
    # only process lines in AsciiDoc files
    @process_lines = ASCIIDOC_EXTENSIONS[::File.extname file]
  else
    @file = nil
    @dir = '.' # right?
    # we don't know what file type we have, so assume AsciiDoc
    @process_lines = true
  end

  if path
    @includes << Helpers.rootname(@path = path)
  else
    @path = '<stdin>'
  end

  @lineno = lineno

  if attributes.key? 'depth'
    depth = attributes['depth'].to_i
    depth = 1 if depth <= 0
    @maxdepth = {:abs => (@include_stack.size - 1) + depth, :rel => depth}
  end

  # effectively fill the buffer
  if (@lines = prepare_lines data, :normalize => true, :condense => false, :indent => attributes['indent']).empty?
    pop_include
  else
    # FIXME we eventually want to handle leveloffset without affecting the lines
    if attributes.key? 'leveloffset'
      @lines.unshift ''
      @lines.unshift %(:leveloffset: #{attributes['leveloffset']})
      @lines << ''
      if (old_leveloffset = @document.attr 'leveloffset')
        @lines << %(:leveloffset: #{old_leveloffset})
      else
        @lines << ':leveloffset!:'
      end
      # compensate for these extra lines
      @lineno -= 2
    end

    # FIXME kind of a hack
    #Document::AttributeEntry.new('infile', @file).save_to_next_block @document
    #Document::AttributeEntry.new('indir', @dir).save_to_next_block @document
    @look_ahead = 0
  end
  self
end
resolve_expr_val(val) click to toggle source

Private: Resolve the value of one side of the expression

Examples

expr = '"value"'
resolve_expr_val expr
# => "value"

expr = '"value'
resolve_expr_val expr
# => "\"value"

expr = '"{undefined}"'
resolve_expr_val expr
# => ""

expr = '{undefined}'
resolve_expr_val expr
# => nil

expr = '2'
resolve_expr_val expr
# => 2

@document.attributes['name'] = 'value'
expr = '"{name}"'
resolve_expr_val expr
# => "value"

Returns The value of the expression, coerced to the appropriate type

# File lib/asciidoctor/reader.rb, line 1192
def resolve_expr_val val
  if ((val.start_with? '"') && (val.end_with? '"')) ||
      ((val.start_with? '\'') && (val.end_with? '\''))
    quoted = true
    val = val[1...-1]
  else
    quoted = false
  end

  # QUESTION should we substitute first?
  # QUESTION should we also require string to be single quoted (like block attribute values?)
  val = @document.sub_attributes val, :attribute_missing => 'drop' if val.include? ATTR_REF_HEAD

  if quoted
    val
  else
    if val.empty?
      nil
    elsif val == 'true'
      true
    elsif val == 'false'
      false
    elsif val.rstrip.empty?
      ' '
    elsif val.include? '.'
      val.to_f
    else
      # fallback to coercing to integer, since we
      # require string values to be explicitly quoted
      val.to_i
    end
  end
end
shift() click to toggle source

TODO Document this override also, we now have the field in the super class, so perhaps just implement the logic there?

Calls superclass method Asciidoctor::Reader#shift
# File lib/asciidoctor/reader.rb, line 1127
def shift
  if @unescape_next_line
    @unescape_next_line = false
    super[1..-1]
  else
    super
  end
end
skip_front_matter!(data, increment_linenos = true) click to toggle source

Private: Ignore front-matter, commonly used in static site generators

# File lib/asciidoctor/reader.rb, line 1137
def skip_front_matter! data, increment_linenos = true
  front_matter = nil
  if data[0] == '---'
    original_data = data.dup
    front_matter = []
    data.shift
    @lineno += 1 if increment_linenos
    while !data.empty? && data[0] != '---'
      front_matter << data.shift
      @lineno += 1 if increment_linenos
    end

    if data.empty?
      data.unshift(*original_data)
      @lineno = 0 if increment_linenos
      front_matter = nil
    else
      data.shift
      @lineno += 1 if increment_linenos
    end
  end

  front_matter
end
to_s() click to toggle source
# File lib/asciidoctor/reader.rb, line 1238
def to_s
  %(#<#{self.class}@#{object_id} {path: #{@path.inspect}, line #: #{@lineno}, include depth: #{@include_stack.size}, include stack: [#{@include_stack.map {|inc| inc.to_s } * ', '}]}>)
end