From 982669812b89432acb05c9ea4a8f8f7728f3fefe Mon Sep 17 00:00:00 2001 From: Kyle Gorman Date: Thu, 25 Apr 2024 13:18:43 -0700 Subject: [PATCH] Internal change. PiperOrigin-RevId: 628170507 --- mozolm/models/ngram_char_fst_model.h | 5 ++--- mozolm/models/ngram_char_fst_model_test.cc | 2 +- mozolm/models/ngram_fst_model.h | 8 +++----- mozolm/models/ngram_word_fst_model.h | 4 ++-- mozolm/models/ngram_word_fst_model_test.cc | 2 +- mozolm/models/ppm_as_fst_model.h | 19 ++++++++----------- mozolm/models/ppm_as_fst_model_test.cc | 4 ++-- mozolm/utils/ngram_fst_relabel.cc | 3 +-- mozolm/utils/ngram_fst_relabel_test.cc | 2 +- 9 files changed, 21 insertions(+), 28 deletions(-) diff --git a/mozolm/models/ngram_char_fst_model.h b/mozolm/models/ngram_char_fst_model.h index 8581e6d..8919cf4 100644 --- a/mozolm/models/ngram_char_fst_model.h +++ b/mozolm/models/ngram_char_fst_model.h @@ -45,14 +45,13 @@ class NGramCharFstModel : public NGramFstModel { // Computes negative log probability for observing the supplied label in a // given state. fst::StdArc::Weight LabelCostInState(fst::StdArc::StateId state, - fst::StdArc::Label label) const; + fst::StdArc::Label label) const; private: fst::StdArc::Label SymLabel(int utf8_sym) const; // Returns negative log probability of the end-of-string at the given state. - fst::StdArc::Weight FinalCostInState( - fst::StdArc::StateId state) const; + fst::StdArc::Weight FinalCostInState(fst::StdArc::StateId state) const; }; } // namespace models diff --git a/mozolm/models/ngram_char_fst_model_test.cc b/mozolm/models/ngram_char_fst_model_test.cc index 3dc3531..eb7374e 100644 --- a/mozolm/models/ngram_char_fst_model_test.cc +++ b/mozolm/models/ngram_char_fst_model_test.cc @@ -34,8 +34,8 @@ #include "nisaba/port/utf8_util.h" #include "nisaba/port/test_utils.h" -using nisaba::testing::TestFilePath; using fst::StdArc; +using nisaba::testing::TestFilePath; namespace mozolm { namespace models { diff --git a/mozolm/models/ngram_fst_model.h b/mozolm/models/ngram_fst_model.h index ce7b3c0..e5b2c95 100644 --- a/mozolm/models/ngram_fst_model.h +++ b/mozolm/models/ngram_fst_model.h @@ -51,9 +51,8 @@ class NGramFstModel : public LanguageModel { // Returns the next state reached by arc labeled with label from state s. // If the label is out-of-vocabulary, it will return the unigram state. - fst::StdArc::StateId NextModelState( - fst::StdArc::StateId current_state, - fst::StdArc::Label label) const; + fst::StdArc::StateId NextModelState(fst::StdArc::StateId current_state, + fst::StdArc::Label label) const; // Language model represented by vector FST. std::unique_ptr fst_; @@ -66,8 +65,7 @@ class NGramFstModel : public LanguageModel { // Checks the current state and sets it to the unigram state if less than // zero. - fst::StdArc::StateId CheckCurrentState( - fst::StdArc::StateId state) const; + fst::StdArc::StateId CheckCurrentState(fst::StdArc::StateId state) const; private: // Performs model sanity check. diff --git a/mozolm/models/ngram_word_fst_model.h b/mozolm/models/ngram_word_fst_model.h index 1a5a764..a62acbf 100644 --- a/mozolm/models/ngram_word_fst_model.h +++ b/mozolm/models/ngram_word_fst_model.h @@ -45,8 +45,8 @@ class NGramImplicitStates { public: NGramImplicitStates() = default; - NGramImplicitStates(const fst::StdVectorFst& fst, - int first_char_begin_index, int first_char_end_index); + NGramImplicitStates(const fst::StdVectorFst& fst, int first_char_begin_index, + int first_char_end_index); // Returns the state if already exists, creates it otherwise. absl::StatusOr GetState(int model_state, int prefix_length, diff --git a/mozolm/models/ngram_word_fst_model_test.cc b/mozolm/models/ngram_word_fst_model_test.cc index 79de42e..01acd83 100644 --- a/mozolm/models/ngram_word_fst_model_test.cc +++ b/mozolm/models/ngram_word_fst_model_test.cc @@ -34,12 +34,12 @@ #include "nisaba/port/test_utils.h" #include "nisaba/port/utf8_util.h" -using ::nisaba::testing::TestFilePath; using ::fst::ArcSort; using ::fst::ILabelCompare; using ::fst::StdArc; using ::fst::StdVectorFst; using ::fst::SymbolTable; +using ::nisaba::testing::TestFilePath; namespace mozolm { namespace models { diff --git a/mozolm/models/ppm_as_fst_model.h b/mozolm/models/ppm_as_fst_model.h index c1a21b4..e340a60 100644 --- a/mozolm/models/ppm_as_fst_model.h +++ b/mozolm/models/ppm_as_fst_model.h @@ -257,9 +257,9 @@ class PpmAsFstModel : public LanguageModel { // Fills in cache vectors of negative log probabilities and destination states // for each item in the vocabulary, matching indices with the symbol table. By // convention, index 0 is for final cost. - absl::Status UpdateCacheAtNonEmptyState( - fst::StdArc::StateId s, fst::StdArc::StateId backoff_state, - const PpmStateCache& backoff_cache); + absl::Status UpdateCacheAtNonEmptyState(fst::StdArc::StateId s, + fst::StdArc::StateId backoff_state, + const PpmStateCache& backoff_cache); // Checks if lower order state caches have updated more recently. bool LowerOrderCacheUpdated(fst::StdArc::StateId s) const; @@ -277,16 +277,14 @@ class PpmAsFstModel : public LanguageModel { absl::StatusOr AddNewState(fst::StdArc::StateId backoff_dest_state); // Returns origin state of arc with symbol from state s. - absl::StatusOr GetArcOriginState(fst::StdArc::StateId s, - int sym_index); + absl::StatusOr GetArcOriginState(fst::StdArc::StateId s, int sym_index); // Returns destination state of arc with symbol from state s. absl::StatusOr GetDestinationState(fst::StdArc::StateId s, int sym_index); // Returns probability of symbol leaving the current state. - absl::StatusOr GetNegLogProb(fst::StdArc::StateId s, - int sym_index); + absl::StatusOr GetNegLogProb(fst::StdArc::StateId s, int sym_index); // Returns normalization value at the current state. absl::StatusOr GetNormalization(fst::StdArc::StateId s); @@ -303,13 +301,12 @@ class PpmAsFstModel : public LanguageModel { // Updates model with an observation of the sym_index at curr_state. absl::StatusOr UpdateModel( - fst::StdArc::StateId curr_state, - fst::StdArc::StateId highest_found_state, int sym_index); + fst::StdArc::StateId curr_state, fst::StdArc::StateId highest_found_state, + int sym_index); // Converts input string into linear FST at the character level, replacing // characters not in possible_characters_ set (if non-empty) with kOovSymbol. - absl::StatusOr String2Fst( - const std::string& input_string); + absl::StatusOr String2Fst(const std::string& input_string); // Adds a single unigram count to every character. absl::Status AddPriorCounts(); diff --git a/mozolm/models/ppm_as_fst_model_test.cc b/mozolm/models/ppm_as_fst_model_test.cc index 9c9c44a..f98ecbc 100644 --- a/mozolm/models/ppm_as_fst_model_test.cc +++ b/mozolm/models/ppm_as_fst_model_test.cc @@ -42,14 +42,14 @@ namespace { constexpr float kFloatDelta = 0.00001; // Delta for float comparisons. constexpr char kVocabFileName[] = "vocab.txt"; -using ::nisaba::file::WriteTempTextFile; -using ::nisaba::utf8::DecodeSingleUnicodeChar; using ::fst::ArcSort; using ::fst::ILabelCompare; using ::fst::Isomorphic; using ::fst::StdArc; using ::fst::StdVectorFst; using ::fst::SymbolTable; +using ::nisaba::file::WriteTempTextFile; +using ::nisaba::utf8::DecodeSingleUnicodeChar; using ::testing::DoubleEq; using ::testing::Each; diff --git a/mozolm/utils/ngram_fst_relabel.cc b/mozolm/utils/ngram_fst_relabel.cc index deb7e25..2c2e688 100644 --- a/mozolm/utils/ngram_fst_relabel.cc +++ b/mozolm/utils/ngram_fst_relabel.cc @@ -136,8 +136,7 @@ absl::Status CheckProperties(const StdVectorFst &fst) { } // namespace absl::Status RelabelWithCodepoints( - const std::vector &keep_symbols_vec, - fst::StdVectorFst *fst) { + const std::vector &keep_symbols_vec, fst::StdVectorFst *fst) { RETURN_IF_ERROR(CheckProperties(*fst)); GOOGLE_LOG(INFO) << "Building input/output mappings and relabeling ..."; const absl::flat_hash_set keep_symbols(keep_symbols_vec.begin(), diff --git a/mozolm/utils/ngram_fst_relabel_test.cc b/mozolm/utils/ngram_fst_relabel_test.cc index 3b8c143..7cda053 100644 --- a/mozolm/utils/ngram_fst_relabel_test.cc +++ b/mozolm/utils/ngram_fst_relabel_test.cc @@ -31,11 +31,11 @@ using fst::ArcIterator; using fst::FstCompiler; +using fst::kError; using fst::StateIterator; using fst::StdArc; using fst::StdVectorFst; using fst::SymbolTable; -using fst::kError; namespace mozolm { namespace {