Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(175)

Side by Side Diff: components/machine_intelligence/ranker_model_loader.cc

Issue 2925733002: Move ranker_model_loader to a new component. (Closed)
Patch Set: Adressing sdefresne's comments. Created 3 years, 5 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
OLDNEW
1 // Copyright 2017 The Chromium Authors. All rights reserved. 1 // Copyright 2017 The Chromium Authors. All rights reserved.
2 // Use of this source code is governed by a BSD-style license that can be 2 // Use of this source code is governed by a BSD-style license that can be
3 // found in the LICENSE file. 3 // found in the LICENSE file.
4 4
5 #include "components/translate/core/browser/ranker_model_loader.h" 5 #include "components/machine_intelligence/ranker_model_loader.h"
6 6
7 #include <utility> 7 #include <utility>
8 8
9 #include "base/bind.h" 9 #include "base/bind.h"
10 #include "base/bind_helpers.h" 10 #include "base/bind_helpers.h"
11 #include "base/command_line.h" 11 #include "base/command_line.h"
12 #include "base/files/file_util.h" 12 #include "base/files/file_util.h"
13 #include "base/files/important_file_writer.h" 13 #include "base/files/important_file_writer.h"
14 #include "base/macros.h" 14 #include "base/macros.h"
15 #include "base/memory/ptr_util.h" 15 #include "base/memory/ptr_util.h"
16 #include "base/metrics/histogram_macros.h" 16 #include "base/metrics/histogram_macros.h"
17 #include "base/profiler/scoped_tracker.h" 17 #include "base/profiler/scoped_tracker.h"
18 #include "base/sequenced_task_runner.h" 18 #include "base/sequenced_task_runner.h"
19 #include "base/strings/string_util.h" 19 #include "base/strings/string_util.h"
20 #include "base/task_runner_util.h" 20 #include "base/task_runner_util.h"
21 #include "base/task_scheduler/post_task.h" 21 #include "base/task_scheduler/post_task.h"
22 #include "base/threading/sequenced_task_runner_handle.h" 22 #include "base/threading/sequenced_task_runner_handle.h"
23 #include "components/translate/core/browser/proto/ranker_model.pb.h" 23 #include "components/machine_intelligence/proto/ranker_model.pb.h"
24 #include "components/translate/core/browser/ranker_model.h" 24 #include "components/machine_intelligence/ranker_model.h"
25 #include "components/translate/core/browser/translate_url_fetcher.h" 25 #include "components/machine_intelligence/ranker_url_fetcher.h"
26 26
27 namespace translate { 27 namespace machine_intelligence {
28 namespace { 28 namespace {
29 29
30 using chrome_intelligence::RankerModel;
31 using chrome_intelligence::RankerModelProto;
32
33 constexpr int kUrlFetcherId = 2;
34
35 // The minimum duration, in minutes, between download attempts. 30 // The minimum duration, in minutes, between download attempts.
36 constexpr int kMinRetryDelayMins = 3; 31 constexpr int kMinRetryDelayMins = 3;
37 32
38 // Suffixes for the various histograms produced by the backend. 33 // Suffixes for the various histograms produced by the backend.
39 const char kWriteTimerHistogram[] = ".Timer.WriteModel"; 34 const char kWriteTimerHistogram[] = ".Timer.WriteModel";
40 const char kReadTimerHistogram[] = ".Timer.ReadModel"; 35 const char kReadTimerHistogram[] = ".Timer.ReadModel";
41 const char kDownloadTimerHistogram[] = ".Timer.DownloadModel"; 36 const char kDownloadTimerHistogram[] = ".Timer.DownloadModel";
42 const char kParsetimerHistogram[] = ".Timer.ParseModel"; 37 const char kParsetimerHistogram[] = ".Timer.ParseModel";
43 const char kModelStatusHistogram[] = ".Model.Status"; 38 const char kModelStatusHistogram[] = ".Model.Status";
44 39
(...skipping 44 matching lines...) Expand 10 before | Expand all | Expand 10 after
89 << model_path.value() << "'."; 84 << model_path.value() << "'.";
90 MyScopedHistogramTimer timer(uma_prefix + kWriteTimerHistogram); 85 MyScopedHistogramTimer timer(uma_prefix + kWriteTimerHistogram);
91 base::ImportantFileWriter::WriteFileAtomically(model_path, model_data); 86 base::ImportantFileWriter::WriteFileAtomically(model_path, model_data);
92 } 87 }
93 88
94 } // namespace 89 } // namespace
95 90
96 RankerModelLoader::RankerModelLoader( 91 RankerModelLoader::RankerModelLoader(
97 ValidateModelCallback validate_model_cb, 92 ValidateModelCallback validate_model_cb,
98 OnModelAvailableCallback on_model_available_cb, 93 OnModelAvailableCallback on_model_available_cb,
94 net::URLRequestContextGetter* request_context_getter,
99 base::FilePath model_path, 95 base::FilePath model_path,
100 GURL model_url, 96 GURL model_url,
101 std::string uma_prefix) 97 std::string uma_prefix)
102 : background_task_runner_(base::CreateSequencedTaskRunnerWithTraits( 98 : background_task_runner_(base::CreateSequencedTaskRunnerWithTraits(
103 {base::MayBlock(), base::TaskPriority::BACKGROUND, 99 {base::MayBlock(), base::TaskPriority::BACKGROUND,
104 base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN})), 100 base::TaskShutdownBehavior::SKIP_ON_SHUTDOWN})),
105 validate_model_cb_(std::move(validate_model_cb)), 101 validate_model_cb_(std::move(validate_model_cb)),
106 on_model_available_cb_(std::move(on_model_available_cb)), 102 on_model_available_cb_(std::move(on_model_available_cb)),
103 request_context_getter_(request_context_getter),
107 model_path_(std::move(model_path)), 104 model_path_(std::move(model_path)),
108 model_url_(std::move(model_url)), 105 model_url_(std::move(model_url)),
109 uma_prefix_(std::move(uma_prefix)), 106 uma_prefix_(std::move(uma_prefix)),
110 url_fetcher_(base::MakeUnique<TranslateURLFetcher>(kUrlFetcherId)), 107 url_fetcher_(base::MakeUnique<RankerURLFetcher>()),
111 weak_ptr_factory_(this) {} 108 weak_ptr_factory_(this) {}
112 109
113 RankerModelLoader::~RankerModelLoader() { 110 RankerModelLoader::~RankerModelLoader() {
114 DCHECK(sequence_checker_.CalledOnValidSequence()); 111 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
115 } 112 }
116 113
117 void RankerModelLoader::NotifyOfRankerActivity() { 114 void RankerModelLoader::NotifyOfRankerActivity() {
118 DCHECK(sequence_checker_.CalledOnValidSequence()); 115 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
119 switch (state_) { 116 switch (state_) {
120 case LoaderState::NOT_STARTED: 117 case LoaderState::NOT_STARTED:
121 if (!model_path_.empty()) { 118 if (!model_path_.empty()) {
122 StartLoadFromFile(); 119 StartLoadFromFile();
123 break; 120 break;
124 } 121 }
125 // There was no configured model path. Switch the state to IDLE and 122 // There was no configured model path. Switch the state to IDLE and
126 // fall through to consider the URL. 123 // fall through to consider the URL.
127 state_ = LoaderState::IDLE; 124 state_ = LoaderState::IDLE;
128 case LoaderState::IDLE: 125 case LoaderState::IDLE:
129 if (model_url_.is_valid()) { 126 if (model_url_.is_valid()) {
130 StartLoadFromURL(); 127 StartLoadFromURL();
131 break; 128 break;
132 } 129 }
133 // There was no configured model URL. Switch the state to FINISHED and 130 // There was no configured model URL. Switch the state to FINISHED and
134 // fall through. 131 // fall through.
135 state_ = LoaderState::FINISHED; 132 state_ = LoaderState::FINISHED;
136 case LoaderState::FINISHED: 133 case LoaderState::FINISHED:
137 case LoaderState::LOADING_FROM_FILE: 134 case LoaderState::LOADING_FROM_FILE:
138 case LoaderState::LOADING_FROM_URL: 135 case LoaderState::LOADING_FROM_URL:
139 // Nothing to do. 136 // Nothing to do.
140 break; 137 break;
141 } 138 }
142 } 139 }
143 140
144 void RankerModelLoader::StartLoadFromFile() { 141 void RankerModelLoader::StartLoadFromFile() {
145 DCHECK(sequence_checker_.CalledOnValidSequence()); 142 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
146 DCHECK_EQ(state_, LoaderState::NOT_STARTED); 143 DCHECK_EQ(state_, LoaderState::NOT_STARTED);
147 DCHECK(!model_path_.empty()); 144 DCHECK(!model_path_.empty());
148 state_ = LoaderState::LOADING_FROM_FILE; 145 state_ = LoaderState::LOADING_FROM_FILE;
149 load_start_time_ = base::TimeTicks::Now(); 146 load_start_time_ = base::TimeTicks::Now();
150 base::PostTaskAndReplyWithResult(background_task_runner_.get(), FROM_HERE, 147 base::PostTaskAndReplyWithResult(background_task_runner_.get(), FROM_HERE,
151 base::Bind(&LoadFromFile, model_path_), 148 base::Bind(&LoadFromFile, model_path_),
152 base::Bind(&RankerModelLoader::OnFileLoaded, 149 base::Bind(&RankerModelLoader::OnFileLoaded,
153 weak_ptr_factory_.GetWeakPtr())); 150 weak_ptr_factory_.GetWeakPtr()));
154 } 151 }
155 152
156 void RankerModelLoader::OnFileLoaded(const std::string& data) { 153 void RankerModelLoader::OnFileLoaded(const std::string& data) {
157 DCHECK(sequence_checker_.CalledOnValidSequence()); 154 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
158 DCHECK_EQ(state_, LoaderState::LOADING_FROM_FILE); 155 DCHECK_EQ(state_, LoaderState::LOADING_FROM_FILE);
159 156
160 // Record the duration of the download. 157 // Record the duration of the download.
161 RecordTimerHistogram(uma_prefix_ + kReadTimerHistogram, 158 RecordTimerHistogram(uma_prefix_ + kReadTimerHistogram,
162 base::TimeTicks::Now() - load_start_time_); 159 base::TimeTicks::Now() - load_start_time_);
163 160
164 // Empty data means |model_path| wasn't successfully read. Otherwise, 161 // Empty data means |model_path| wasn't successfully read. Otherwise,
165 // parse and validate the model. 162 // parse and validate the model.
166 std::unique_ptr<RankerModel> model; 163 std::unique_ptr<RankerModel> model;
167 if (data.empty()) { 164 if (data.empty()) {
(...skipping 26 matching lines...) Expand all
194 on_model_available_cb_.Run(std::move(model)); 191 on_model_available_cb_.Run(std::move(model));
195 } 192 }
196 193
197 // Notify the state machine. This will immediately kick off a download if 194 // Notify the state machine. This will immediately kick off a download if
198 // one is required, instead of waiting for the next organic detection of 195 // one is required, instead of waiting for the next organic detection of
199 // ranker activity. 196 // ranker activity.
200 NotifyOfRankerActivity(); 197 NotifyOfRankerActivity();
201 } 198 }
202 199
203 void RankerModelLoader::StartLoadFromURL() { 200 void RankerModelLoader::StartLoadFromURL() {
204 DCHECK(sequence_checker_.CalledOnValidSequence()); 201 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
205 DCHECK_EQ(state_, LoaderState::IDLE); 202 DCHECK_EQ(state_, LoaderState::IDLE);
206 DCHECK(model_url_.is_valid()); 203 DCHECK(model_url_.is_valid());
207 204
208 // Do nothing if the download attempts should be throttled. 205 // Do nothing if the download attempts should be throttled.
209 if (base::TimeTicks::Now() < next_earliest_download_time_) { 206 if (base::TimeTicks::Now() < next_earliest_download_time_) {
210 DVLOG(2) << "Last download attempt was too recent."; 207 DVLOG(2) << "Last download attempt was too recent.";
211 ReportModelStatus(RankerModelStatus::DOWNLOAD_THROTTLED); 208 ReportModelStatus(RankerModelStatus::DOWNLOAD_THROTTLED);
212 return; 209 return;
213 } 210 }
214 211
215 // Kick off the next download attempt and reset the time of the next earliest 212 // Kick off the next download attempt and reset the time of the next earliest
216 // allowable download attempt. 213 // allowable download attempt.
217 DVLOG(2) << "Downloading model from: " << model_url_; 214 DVLOG(2) << "Downloading model from: " << model_url_;
218 state_ = LoaderState::LOADING_FROM_URL; 215 state_ = LoaderState::LOADING_FROM_URL;
219 load_start_time_ = base::TimeTicks::Now(); 216 load_start_time_ = base::TimeTicks::Now();
220 next_earliest_download_time_ = 217 next_earliest_download_time_ =
221 load_start_time_ + base::TimeDelta::FromMinutes(kMinRetryDelayMins); 218 load_start_time_ + base::TimeDelta::FromMinutes(kMinRetryDelayMins);
222 bool request_started = url_fetcher_->Request( 219 bool request_started =
223 model_url_, base::Bind(&RankerModelLoader::OnURLFetched, 220 url_fetcher_->Request(model_url_,
224 weak_ptr_factory_.GetWeakPtr())); 221 base::Bind(&RankerModelLoader::OnURLFetched,
222 weak_ptr_factory_.GetWeakPtr()),
223 request_context_getter_.get());
225 224
226 // |url_fetcher_| maintains a request retry counter. If all allowed attempts 225 // |url_fetcher_| maintains a request retry counter. If all allowed attempts
227 // have already been exhausted, then the loader is finished and has abandoned 226 // have already been exhausted, then the loader is finished and has abandoned
228 // loading the model. 227 // loading the model.
229 if (!request_started) { 228 if (!request_started) {
230 DVLOG(2) << "Model download abandoned."; 229 DVLOG(2) << "Model download abandoned.";
231 ReportModelStatus(RankerModelStatus::MODEL_LOADING_ABANDONED); 230 ReportModelStatus(RankerModelStatus::MODEL_LOADING_ABANDONED);
232 state_ = LoaderState::FINISHED; 231 state_ = LoaderState::FINISHED;
233 } 232 }
234 } 233 }
235 234
236 void RankerModelLoader::OnURLFetched(int /* id */, 235 void RankerModelLoader::OnURLFetched(bool success, const std::string& data) {
237 bool success, 236 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
238 const std::string& data) {
239 DCHECK(sequence_checker_.CalledOnValidSequence());
240 DCHECK_EQ(state_, LoaderState::LOADING_FROM_URL); 237 DCHECK_EQ(state_, LoaderState::LOADING_FROM_URL);
241 238
242 // Record the duration of the download. 239 // Record the duration of the download.
243 RecordTimerHistogram(uma_prefix_ + kDownloadTimerHistogram, 240 RecordTimerHistogram(uma_prefix_ + kDownloadTimerHistogram,
244 base::TimeTicks::Now() - load_start_time_); 241 base::TimeTicks::Now() - load_start_time_);
245 242
246 // On request failure, transition back to IDLE. The loader will retry, or 243 // On request failure, transition back to IDLE. The loader will retry, or
247 // enforce the max download attempts, later. 244 // enforce the max download attempts, later.
248 if (!success || data.empty()) { 245 if (!success || data.empty()) {
249 DVLOG(2) << "Download from '" << model_url_ << "'' failed."; 246 DVLOG(2) << "Download from '" << model_url_ << "'' failed.";
(...skipping 23 matching lines...) Expand all
273 background_task_runner_->PostTask( 270 background_task_runner_->PostTask(
274 FROM_HERE, base::BindOnce(&SaveToFile, model_url_, model_path_, 271 FROM_HERE, base::BindOnce(&SaveToFile, model_url_, model_path_,
275 model->SerializeAsString(), uma_prefix_)); 272 model->SerializeAsString(), uma_prefix_));
276 } 273 }
277 274
278 // The loader is finished. Transfer the model to the client. 275 // The loader is finished. Transfer the model to the client.
279 state_ = LoaderState::FINISHED; 276 state_ = LoaderState::FINISHED;
280 on_model_available_cb_.Run(std::move(model)); 277 on_model_available_cb_.Run(std::move(model));
281 } 278 }
282 279
283 std::unique_ptr<chrome_intelligence::RankerModel> 280 std::unique_ptr<RankerModel> RankerModelLoader::CreateAndValidateModel(
284 RankerModelLoader::CreateAndValidateModel(const std::string& data) { 281 const std::string& data) {
285 DCHECK(sequence_checker_.CalledOnValidSequence()); 282 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
286 MyScopedHistogramTimer timer(uma_prefix_ + kParsetimerHistogram); 283 MyScopedHistogramTimer timer(uma_prefix_ + kParsetimerHistogram);
287 auto model = RankerModel::FromString(data); 284 auto model = RankerModel::FromString(data);
288 if (ReportModelStatus(model ? validate_model_cb_.Run(*model) 285 if (ReportModelStatus(model ? validate_model_cb_.Run(*model)
289 : RankerModelStatus::PARSE_FAILED) != 286 : RankerModelStatus::PARSE_FAILED) !=
290 RankerModelStatus::OK) { 287 RankerModelStatus::OK) {
291 return nullptr; 288 return nullptr;
292 } 289 }
293 return model; 290 return model;
294 } 291 }
295 292
296 RankerModelStatus RankerModelLoader::ReportModelStatus( 293 RankerModelStatus RankerModelLoader::ReportModelStatus(
297 RankerModelStatus model_status) { 294 RankerModelStatus model_status) {
298 DCHECK(sequence_checker_.CalledOnValidSequence()); 295 DCHECK_CALLED_ON_VALID_SEQUENCE(sequence_checker_);
299 base::HistogramBase* histogram = base::LinearHistogram::FactoryGet( 296 base::HistogramBase* histogram = base::LinearHistogram::FactoryGet(
300 uma_prefix_ + kModelStatusHistogram, 1, 297 uma_prefix_ + kModelStatusHistogram, 1,
301 static_cast<int>(RankerModelStatus::MAX), 298 static_cast<int>(RankerModelStatus::MAX),
302 static_cast<int>(RankerModelStatus::MAX) + 1, 299 static_cast<int>(RankerModelStatus::MAX) + 1,
303 base::HistogramBase::kUmaTargetedHistogramFlag); 300 base::HistogramBase::kUmaTargetedHistogramFlag);
304 if (histogram) 301 if (histogram)
305 histogram->Add(static_cast<int>(model_status)); 302 histogram->Add(static_cast<int>(model_status));
306 return model_status; 303 return model_status;
307 } 304 }
308 305
309 } // namespace translate 306 } // namespace machine_intelligence
OLDNEW
« no previous file with comments | « components/machine_intelligence/ranker_model_loader.h ('k') | components/machine_intelligence/ranker_model_loader_unittest.cc » ('j') | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698