This class is responsible for arranging examples in groups that have the same strucure (i.e. More...

#include <nnet-discriminative-example.h>

Collaboration diagram for DiscriminativeExampleMerger:

[legend]

Public Member Functions
	DiscriminativeExampleMerger (const ExampleMergingConfig &config, NnetDiscriminativeExampleWriter *writer)

void	AcceptExample (NnetDiscriminativeExample *a)

void	Finish ()

int32	ExitStatus ()

	~DiscriminativeExampleMerger ()

Private Types
typedef unordered_map< NnetDiscriminativeExample , std::vector< NnetDiscriminativeExample >, NnetDiscriminativeExampleStructureHasher, NnetDiscriminativeExampleStructureCompare >	MapType

Private Member Functions
void	WriteMinibatch (std::vector< NnetDiscriminativeExample > *egs)

Private Attributes
bool	finished_

int32	num_egs_written_

const ExampleMergingConfig &	config_

NnetDiscriminativeExampleWriter *	writer_

ExampleMergingStats	stats_

MapType	eg_to_egs_

Detailed Description

This class is responsible for arranging examples in groups that have the same strucure (i.e.

the same input and output indexes), and outputting them in suitable minibatches as defined by ExampleMergingConfig.

Definition at line 228 of file nnet-discriminative-example.h.

Member Typedef Documentation

◆ MapType

typedef unordered_map<NnetDiscriminativeExample*, std::vector<NnetDiscriminativeExample*>, NnetDiscriminativeExampleStructureHasher, NnetDiscriminativeExampleStructureCompare> MapType

private

Definition at line 266 of file nnet-discriminative-example.h.

Constructor & Destructor Documentation

◆ DiscriminativeExampleMerger()

DiscriminativeExampleMerger	(	const ExampleMergingConfig &	config,
		NnetDiscriminativeExampleWriter *	writer
	)

Definition at line 446 of file nnet-discriminative-example.cc.

                                                                      :
     finished_(false), num_egs_written_(0),
     config_(config), writer_(writer) { }

◆ ~DiscriminativeExampleMerger()

~DiscriminativeExampleMerger ( )

inline

Definition at line 248 of file nnet-discriminative-example.h.

248 { Finish(); };

kaldi::nnet3::DiscriminativeExampleMerger::Finish

void Finish()

Definition: nnet-discriminative-example.cc:498

Member Function Documentation

◆ AcceptExample()

void AcceptExample ( NnetDiscriminativeExample * a )

Definition at line 452 of file nnet-discriminative-example.cc.

References DiscriminativeExampleMerger::config_, DiscriminativeExampleMerger::eg_to_egs_, DiscriminativeExampleMerger::finished_, kaldi::nnet3::GetNnetDiscriminativeExampleSize(), rnnlm::i, KALDI_ASSERT, ExampleMergingConfig::MinibatchSize(), and DiscriminativeExampleMerger::WriteMinibatch().

Referenced by main().

                                                                              {
   KALDI_ASSERT(!finished_);
   // If an eg with the same structure as 'eg' is already a key in the
   // map, it won't be replaced, but if it's new it will be made
   // the key.  Also we remove the key before making the vector empty.
   // This way we ensure that the eg in the key is always the first
   // element of the vector.
   std::vector<NnetDiscriminativeExample*> &vec = eg_to_egs_[eg];
   vec.push_back(eg);
   int32 eg_size = GetNnetDiscriminativeExampleSize(*eg),
       num_available = vec.size();
   bool input_ended = false;
   int32 minibatch_size = config_.MinibatchSize(eg_size, num_available,
                                                input_ended);
   if (minibatch_size != 0) {  // we need to write out a merged eg.
     KALDI_ASSERT(minibatch_size == num_available);
 
     std::vector<NnetDiscriminativeExample*> vec_copy(vec);
     eg_to_egs_.erase(eg);
 
     // MergeDiscriminativeExamples() expects a vector of NnetDiscriminativeExample, not of pointers,
     // so use swap to create that without doing any real work.
     std::vector<NnetDiscriminativeExample> egs_to_merge(minibatch_size);
     for (int32 i = 0; i < minibatch_size; i++) {
       egs_to_merge[i].Swap(vec_copy[i]);
       delete vec_copy[i];  // we owned those pointers.
     }
     WriteMinibatch(&egs_to_merge);
   }
 }

◆ ExitStatus()

int32 ExitStatus ( )

inline

Definition at line 246 of file nnet-discriminative-example.h.

Referenced by main().

246 { Finish(); return (num_egs_written_ > 0 ? 0 : 1); }

kaldi::nnet3::DiscriminativeExampleMerger::Finish

void Finish()

Definition: nnet-discriminative-example.cc:498

kaldi::nnet3::DiscriminativeExampleMerger::num_egs_written_

int32 num_egs_written_

Definition: nnet-discriminative-example.h:257

◆ Finish()

void Finish ( )

Definition at line 498 of file nnet-discriminative-example.cc.

References DiscriminativeExampleMerger::config_, ExampleMergingStats::DiscardedExamples(), DiscriminativeExampleMerger::eg_to_egs_, DiscriminativeExampleMerger::finished_, kaldi::nnet3::GetNnetDiscriminativeExampleSize(), rnnlm::i, KALDI_ASSERT, ExampleMergingConfig::MinibatchSize(), ExampleMergingStats::PrintStats(), DiscriminativeExampleMerger::stats_, and DiscriminativeExampleMerger::WriteMinibatch().

Referenced by main().

                                          {
   if (finished_) return;  // already finished.
   finished_ = true;
 
   // we'll convert the map eg_to_egs_ to a vector of vectors to avoid
   // iterator invalidation problems.
   std::vector<std::vector<NnetDiscriminativeExample*> > all_egs;
   all_egs.reserve(eg_to_egs_.size());
 
   MapType::iterator iter = eg_to_egs_.begin(), end = eg_to_egs_.end();
   for (; iter != end; ++iter)
     all_egs.push_back(iter->second);
   eg_to_egs_.clear();
 
   for (size_t i = 0; i < all_egs.size(); i++) {
     int32 minibatch_size;
     std::vector<NnetDiscriminativeExample*> &vec = all_egs[i];
     KALDI_ASSERT(!vec.empty());
     int32 eg_size = GetNnetDiscriminativeExampleSize(*(vec[0]));
     bool input_ended = true;
     while (!vec.empty() &&
            (minibatch_size = config_.MinibatchSize(eg_size, vec.size(),
                                                    input_ended)) != 0) {
       // MergeDiscriminativeExamples() expects a vector of
       // NnetDiscriminativeExample, not of pointers, so use swap to create that
       // without doing any real work.
       std::vector<NnetDiscriminativeExample> egs_to_merge(minibatch_size);
       for (int32 i = 0; i < minibatch_size; i++) {
         egs_to_merge[i].Swap(vec[i]);
         delete vec[i];  // we owned those pointers.
       }
       vec.erase(vec.begin(), vec.begin() + minibatch_size);
       WriteMinibatch(&egs_to_merge);
     }
     if (!vec.empty()) {
       int32 eg_size = GetNnetDiscriminativeExampleSize(*(vec[0]));
       NnetDiscriminativeExampleStructureHasher eg_hasher;
       size_t structure_hash = eg_hasher(*(vec[0]));
       int32 num_discarded = vec.size();
       stats_.DiscardedExamples(eg_size, structure_hash, num_discarded);
       for (int32 i = 0; i < num_discarded; i++)
         delete vec[i];
       vec.clear();
     }
   }
   stats_.PrintStats();
 }

◆ WriteMinibatch()

void WriteMinibatch ( std::vector< NnetDiscriminativeExample > * egs )

private

Definition at line 483 of file nnet-discriminative-example.cc.

References ExampleMergingConfig::compress, DiscriminativeExampleMerger::config_, kaldi::nnet3::GetNnetDiscriminativeExampleSize(), KALDI_ASSERT, kaldi::nnet3::MergeDiscriminativeExamples(), DiscriminativeExampleMerger::num_egs_written_, DiscriminativeExampleMerger::stats_, TableWriter< Holder >::Write(), DiscriminativeExampleMerger::writer_, and ExampleMergingStats::WroteExample().

Referenced by DiscriminativeExampleMerger::AcceptExample(), and DiscriminativeExampleMerger::Finish().

                                                {
   KALDI_ASSERT(!egs->empty());
   int32 eg_size = GetNnetDiscriminativeExampleSize((*egs)[0]);
   NnetDiscriminativeExampleStructureHasher eg_hasher;
   size_t structure_hash = eg_hasher((*egs)[0]);
   int32 minibatch_size = egs->size();
   stats_.WroteExample(eg_size, structure_hash, minibatch_size);
   NnetDiscriminativeExample merged_eg;
   MergeDiscriminativeExamples(config_.compress, egs, &merged_eg);
   std::ostringstream key;
   key << "merged-" << (num_egs_written_++) << "-" << minibatch_size;
   writer_->Write(key.str(), merged_eg);
 }