kenlm/lm/common/joint_order.hh

#ifndef LM_COMMON_JOINT_ORDER_H
#define LM_COMMON_JOINT_ORDER_H

#include "lm/common/ngram_stream.hh"
#include "lm/lm_exception.hh"

#ifdef DEBUG
#include "util/fixed_array.hh"
#include <iostream>
#endif

#include <cstring>

namespace lm {

template <class Callback, class Compare> void JointOrder(const util::stream::ChainPositions &positions, Callback &callback) {
  // Allow matching to reference streams[-1].
  util::FixedArray<ProxyStream<NGramHeader> > streams_with_dummy(positions.size() + 1);
  // A bogus stream for [-1].
  streams_with_dummy.push_back();
  for (std::size_t i = 0; i < positions.size(); ++i) {
    streams_with_dummy.push_back(positions[i], NGramHeader(NULL, i + 1));
  }
  ProxyStream<NGramHeader> *streams = streams_with_dummy.begin() + 1;

  std::size_t order;
  for (order = 0; order < positions.size() && streams[order]; ++order) {}
  assert(order); // should always have <unk>.

  // Debugging only: call comparison function to sanity check order.
#ifdef DEBUG
  util::FixedArray<Compare> less_compare(order);
  for (unsigned i = 0; i < order; ++i)
    less_compare.push_back(i + 1);
#endif // DEBUG

  std::size_t current = 0;
  while (true) {
    // Does the context match the lower one?
    if (!memcmp(streams[static_cast<int>(current) - 1]->begin(), streams[current]->begin() + Compare::kMatchOffset, sizeof(WordIndex) * current)) {
      callback.Enter(current, streams[current].Get());
      // Transition to looking for extensions.
      if (++current < order) continue;
    }
#ifdef DEBUG
    // match_check[current - 1] matches current-grams
    // The lower-order stream (which skips fewer current-grams) should always be <= the higher order-stream (which can skip current-grams).
    else if (!less_compare[current - 1](streams[static_cast<int>(current) - 1]->begin(), streams[current]->begin() + Compare::kMatchOffset)) {
      std::cerr << "Stream out of order detected" << std::endl;
      abort();
    }
#endif // DEBUG
    // No extension left.
    while(true) {
      assert(current > 0);
      --current;
      callback.Exit(current, streams[current].Get());

      if (++streams[current]) break;

      UTIL_THROW_IF(order != current + 1, FormatLoadException, "Detected n-gram without matching suffix");

      order = current;
      if (!order) return;
    }
  }
}

} // namespaces

#endif // LM_COMMON_JOINT_ORDER_H