Chromium Code Reviews
chromiumcodereview-hr@appspot.gserviceaccount.com (chromiumcodereview-hr) | Please choose your nickname with Settings | Help | Chromium Project | Gerrit Changes | Sign out
(11)

Side by Side Diff: telemetry/telemetry/wpr/archive_info.py

Issue 3016693002: Remove support for legacy wpr_server in Telemetry
Patch Set: Created 3 years, 2 months ago
Use n/p to move between diff chunks; N/P to move between comments. Draft comments are only viewable by you.
Jump to:
View unified diff | Download patch
« no previous file with comments | « telemetry/telemetry/testing/serially_executed_browser_test_case.py ('k') | no next file » | no next file with comments »
Toggle Intra-line Diffs ('i') | Expand Comments ('e') | Collapse Comments ('c') | Show Comments Hide Comments ('s')
OLDNEW
1 # Copyright 2013 The Chromium Authors. All rights reserved. 1 # Copyright 2013 The Chromium Authors. All rights reserved.
2 # Use of this source code is governed by a BSD-style license that can be 2 # Use of this source code is governed by a BSD-style license that can be
3 # found in the LICENSE file. 3 # found in the LICENSE file.
4 4
5 import json 5 import json
6 import logging 6 import logging
7 import os 7 import os
8 import re 8 import re
9 import shutil 9 import shutil
10 import tempfile 10 import tempfile
(...skipping 24 matching lines...) Expand all
35 self._bucket = bucket 35 self._bucket = bucket
36 self.temp_target_wpr_file_path = None 36 self.temp_target_wpr_file_path = None
37 # Ensure directory exists. 37 # Ensure directory exists.
38 if not os.path.exists(self._base_dir): 38 if not os.path.exists(self._base_dir):
39 os.makedirs(self._base_dir) 39 os.makedirs(self._base_dir)
40 40
41 assert data.get('platform_specific', False), ( 41 assert data.get('platform_specific', False), (
42 'Detected old version of archive info json file. Please update to new ' 42 'Detected old version of archive info json file. Please update to new '
43 'version.') 43 'version.')
44 44
45 self.is_using_wpr_go_archives = True
46 self._story_name_to_wpr_file = data['archives'] 45 self._story_name_to_wpr_file = data['archives']
47 story_archives = self._data['archives']
48 for story in story_archives:
49 files = story_archives[story]
50 for f in files:
51 if files[f].endswith('.wpr'):
52 self.is_using_wpr_go_archives = False
53 self.ValidateArchivesFormat(story_archives)
54
55 @staticmethod
56 def ValidateArchivesFormat(story_archives):
57 """ Checks that all archives follow either .wpr format or .wprgo format
58 but not both.
59 """
60 using_wpr = False
61 using_wpr_go = False
62 for story in story_archives:
63 files = story_archives[story]
64 for f in files:
65 if files[f].endswith('.wprgo'):
66 using_wpr_go = True
67 elif files[f].endswith('.wpr'):
68 using_wpr = True
69 assert not(using_wpr and using_wpr_go), (
70 'Detected both .wprgo or .wpr archive format.')
71 46
72 @classmethod 47 @classmethod
73 def FromFile(cls, file_path, bucket): 48 def FromFile(cls, file_path, bucket):
74 """ Generates an archive_info instance with the given json file. """ 49 """ Generates an archive_info instance with the given json file. """
75 if os.path.exists(file_path): 50 if os.path.exists(file_path):
76 with open(file_path, 'r') as f: 51 with open(file_path, 'r') as f:
77 data = json.load(f) 52 data = json.load(f)
78 return cls(file_path, data, bucket) 53 return cls(file_path, data, bucket)
79 return cls(file_path, {'archives': {}, 'platform_specific': True}, bucket) 54 return cls(file_path, {'archives': {}, 'platform_specific': True}, bucket)
80 55
(...skipping 10 matching lines...) Expand all
91 exists. 66 exists.
92 """ 67 """
93 logging.info('Downloading WPR archives. This can take a long time.') 68 logging.info('Downloading WPR archives. This can take a long time.')
94 start_time = time.time() 69 start_time = time.time()
95 # If no target platform is set, download all platforms. 70 # If no target platform is set, download all platforms.
96 if target_platforms is None: 71 if target_platforms is None:
97 target_platforms = _ALL_PLATFORMS 72 target_platforms = _ALL_PLATFORMS
98 else: 73 else:
99 assert isinstance(target_platforms, list), 'Must pass platforms as a list' 74 assert isinstance(target_platforms, list), 'Must pass platforms as a list'
100 target_platforms = target_platforms + [_DEFAULT_PLATFORM] 75 target_platforms = target_platforms + [_DEFAULT_PLATFORM]
101 # Download all .wpr or .wprgo files. 76 # Download all .wprgo files.
102 if not self._bucket: 77 if not self._bucket:
103 logging.warning('Story set in %s has no bucket specified, and ' 78 logging.warning('Story set in %s has no bucket specified, and '
104 'cannot be downloaded from cloud_storage.', ) 79 'cannot be downloaded from cloud_storage.', )
105 return 80 return
106 assert 'archives' in self._data, ("Invalid data format in %s. 'archives' " 81 assert 'archives' in self._data, ("Invalid data format in %s. 'archives' "
107 "field is needed" % self._file_path) 82 "field is needed" % self._file_path)
108 83
109 def download_if_needed(path): 84 def download_if_needed(path):
110 try: 85 try:
111 cloud_storage.GetIfChanged(path, self._bucket) 86 cloud_storage.GetIfChanged(path, self._bucket)
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after
209 def _WprFileNameToPath(self, wpr_file): 184 def _WprFileNameToPath(self, wpr_file):
210 return os.path.abspath(os.path.join(self._base_dir, wpr_file)) 185 return os.path.abspath(os.path.join(self._base_dir, wpr_file))
211 186
212 def _NextWprFileName(self): 187 def _NextWprFileName(self):
213 """Creates a new file name for a wpr archive file.""" 188 """Creates a new file name for a wpr archive file."""
214 # The names are of the format "some_thing_number.wpr" or 189 # The names are of the format "some_thing_number.wpr" or
215 # "some_thing_number.wprgo". Read the numbers. 190 # "some_thing_number.wprgo". Read the numbers.
216 highest_number = -1 191 highest_number = -1
217 base = None 192 base = None
218 wpr_files = [] 193 wpr_files = []
219 extension = 'wprgo' if self.is_using_wpr_go_archives else 'wpr' 194 extension = 'wprgo'
220 for story in self._data['archives']: 195 for story in self._data['archives']:
221 for p in self._data['archives'][story]: 196 for p in self._data['archives'][story]:
222 wpr_files.append(self._data['archives'][story][p]) 197 wpr_files.append(self._data['archives'][story][p])
223 198
224 for wpr_file in wpr_files: 199 for wpr_file in wpr_files:
225 pattern = r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+).{extension}'.format( 200 pattern = r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+).{extension}'.format(
226 extension=extension) 201 extension=extension)
227 match = re.match(pattern, wpr_file) 202 match = re.match(pattern, wpr_file)
228 if not match: 203 if not match:
229 raise Exception('Illegal wpr file name ' + wpr_file) 204 raise Exception('Illegal wpr file name ' + wpr_file)
230 highest_number = max(int(match.groupdict()['NUMBER']), highest_number) 205 highest_number = max(int(match.groupdict()['NUMBER']), highest_number)
231 if base and match.groupdict()['BASE'] != base: 206 if base and match.groupdict()['BASE'] != base:
232 raise Exception('Illegal wpr file name ' + wpr_file + 207 raise Exception('Illegal wpr file name ' + wpr_file +
233 ', doesn\'t begin with ' + base) 208 ', doesn\'t begin with ' + base)
234 base = match.groupdict()['BASE'] 209 base = match.groupdict()['BASE']
235 if not base: 210 if not base:
236 # If we're creating a completely new info file, use the base name of the 211 # If we're creating a completely new info file, use the base name of the
237 # story set file. 212 # story set file.
238 base = os.path.splitext(os.path.basename(self._file_path))[0] 213 base = os.path.splitext(os.path.basename(self._file_path))[0]
239 new_filename = '%s_%03d.%s' % (base, highest_number + 1, extension) 214 new_filename = '%s_%03d.%s' % (base, highest_number + 1, extension)
240 return new_filename, self._WprFileNameToPath(new_filename) 215 return new_filename, self._WprFileNameToPath(new_filename)
241 216
242 def _SetWprFileForStory(self, story_name, wpr_file, target_platform): 217 def _SetWprFileForStory(self, story_name, wpr_file, target_platform):
243 """For modifying the metadata when we're going to record a new archive.""" 218 """For modifying the metadata when we're going to record a new archive."""
244 if story_name not in self._data['archives']: 219 if story_name not in self._data['archives']:
245 # If there is no other recording we want the first to be the default 220 # If there is no other recording we want the first to be the default
246 # until a new default is recorded. 221 # until a new default is recorded.
247 self._data['archives'][story_name] = {_DEFAULT_PLATFORM: wpr_file} 222 self._data['archives'][story_name] = {_DEFAULT_PLATFORM: wpr_file}
248 self._data['archives'][story_name][target_platform] = wpr_file 223 self._data['archives'][story_name][target_platform] = wpr_file
OLDNEW
« no previous file with comments | « telemetry/telemetry/testing/serially_executed_browser_test_case.py ('k') | no next file » | no next file with comments »

Powered by Google App Engine
This is Rietveld 408576698