OLD | NEW |
1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
4 | 4 |
5 import json | 5 import json |
6 import logging | 6 import logging |
7 import os | 7 import os |
8 import re | 8 import re |
9 import shutil | 9 import shutil |
10 import tempfile | 10 import tempfile |
(...skipping 24 matching lines...) Expand all Loading... |
35 self._bucket = bucket | 35 self._bucket = bucket |
36 self.temp_target_wpr_file_path = None | 36 self.temp_target_wpr_file_path = None |
37 # Ensure directory exists. | 37 # Ensure directory exists. |
38 if not os.path.exists(self._base_dir): | 38 if not os.path.exists(self._base_dir): |
39 os.makedirs(self._base_dir) | 39 os.makedirs(self._base_dir) |
40 | 40 |
41 assert data.get('platform_specific', False), ( | 41 assert data.get('platform_specific', False), ( |
42 'Detected old version of archive info json file. Please update to new ' | 42 'Detected old version of archive info json file. Please update to new ' |
43 'version.') | 43 'version.') |
44 | 44 |
45 self.is_using_wpr_go_archives = True | |
46 self._story_name_to_wpr_file = data['archives'] | 45 self._story_name_to_wpr_file = data['archives'] |
47 story_archives = self._data['archives'] | |
48 for story in story_archives: | |
49 files = story_archives[story] | |
50 for f in files: | |
51 if files[f].endswith('.wpr'): | |
52 self.is_using_wpr_go_archives = False | |
53 self.ValidateArchivesFormat(story_archives) | |
54 | |
55 @staticmethod | |
56 def ValidateArchivesFormat(story_archives): | |
57 """ Checks that all archives follow either .wpr format or .wprgo format | |
58 but not both. | |
59 """ | |
60 using_wpr = False | |
61 using_wpr_go = False | |
62 for story in story_archives: | |
63 files = story_archives[story] | |
64 for f in files: | |
65 if files[f].endswith('.wprgo'): | |
66 using_wpr_go = True | |
67 elif files[f].endswith('.wpr'): | |
68 using_wpr = True | |
69 assert not(using_wpr and using_wpr_go), ( | |
70 'Detected both .wprgo or .wpr archive format.') | |
71 | 46 |
72 @classmethod | 47 @classmethod |
73 def FromFile(cls, file_path, bucket): | 48 def FromFile(cls, file_path, bucket): |
74 """ Generates an archive_info instance with the given json file. """ | 49 """ Generates an archive_info instance with the given json file. """ |
75 if os.path.exists(file_path): | 50 if os.path.exists(file_path): |
76 with open(file_path, 'r') as f: | 51 with open(file_path, 'r') as f: |
77 data = json.load(f) | 52 data = json.load(f) |
78 return cls(file_path, data, bucket) | 53 return cls(file_path, data, bucket) |
79 return cls(file_path, {'archives': {}, 'platform_specific': True}, bucket) | 54 return cls(file_path, {'archives': {}, 'platform_specific': True}, bucket) |
80 | 55 |
(...skipping 10 matching lines...) Expand all Loading... |
91 exists. | 66 exists. |
92 """ | 67 """ |
93 logging.info('Downloading WPR archives. This can take a long time.') | 68 logging.info('Downloading WPR archives. This can take a long time.') |
94 start_time = time.time() | 69 start_time = time.time() |
95 # If no target platform is set, download all platforms. | 70 # If no target platform is set, download all platforms. |
96 if target_platforms is None: | 71 if target_platforms is None: |
97 target_platforms = _ALL_PLATFORMS | 72 target_platforms = _ALL_PLATFORMS |
98 else: | 73 else: |
99 assert isinstance(target_platforms, list), 'Must pass platforms as a list' | 74 assert isinstance(target_platforms, list), 'Must pass platforms as a list' |
100 target_platforms = target_platforms + [_DEFAULT_PLATFORM] | 75 target_platforms = target_platforms + [_DEFAULT_PLATFORM] |
101 # Download all .wpr or .wprgo files. | 76 # Download all .wprgo files. |
102 if not self._bucket: | 77 if not self._bucket: |
103 logging.warning('Story set in %s has no bucket specified, and ' | 78 logging.warning('Story set in %s has no bucket specified, and ' |
104 'cannot be downloaded from cloud_storage.', ) | 79 'cannot be downloaded from cloud_storage.', ) |
105 return | 80 return |
106 assert 'archives' in self._data, ("Invalid data format in %s. 'archives' " | 81 assert 'archives' in self._data, ("Invalid data format in %s. 'archives' " |
107 "field is needed" % self._file_path) | 82 "field is needed" % self._file_path) |
108 | 83 |
109 def download_if_needed(path): | 84 def download_if_needed(path): |
110 try: | 85 try: |
111 cloud_storage.GetIfChanged(path, self._bucket) | 86 cloud_storage.GetIfChanged(path, self._bucket) |
(...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
209 def _WprFileNameToPath(self, wpr_file): | 184 def _WprFileNameToPath(self, wpr_file): |
210 return os.path.abspath(os.path.join(self._base_dir, wpr_file)) | 185 return os.path.abspath(os.path.join(self._base_dir, wpr_file)) |
211 | 186 |
212 def _NextWprFileName(self): | 187 def _NextWprFileName(self): |
213 """Creates a new file name for a wpr archive file.""" | 188 """Creates a new file name for a wpr archive file.""" |
214 # The names are of the format "some_thing_number.wpr" or | 189 # The names are of the format "some_thing_number.wpr" or |
215 # "some_thing_number.wprgo". Read the numbers. | 190 # "some_thing_number.wprgo". Read the numbers. |
216 highest_number = -1 | 191 highest_number = -1 |
217 base = None | 192 base = None |
218 wpr_files = [] | 193 wpr_files = [] |
219 extension = 'wprgo' if self.is_using_wpr_go_archives else 'wpr' | 194 extension = 'wprgo' |
220 for story in self._data['archives']: | 195 for story in self._data['archives']: |
221 for p in self._data['archives'][story]: | 196 for p in self._data['archives'][story]: |
222 wpr_files.append(self._data['archives'][story][p]) | 197 wpr_files.append(self._data['archives'][story][p]) |
223 | 198 |
224 for wpr_file in wpr_files: | 199 for wpr_file in wpr_files: |
225 pattern = r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+).{extension}'.format( | 200 pattern = r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+).{extension}'.format( |
226 extension=extension) | 201 extension=extension) |
227 match = re.match(pattern, wpr_file) | 202 match = re.match(pattern, wpr_file) |
228 if not match: | 203 if not match: |
229 raise Exception('Illegal wpr file name ' + wpr_file) | 204 raise Exception('Illegal wpr file name ' + wpr_file) |
230 highest_number = max(int(match.groupdict()['NUMBER']), highest_number) | 205 highest_number = max(int(match.groupdict()['NUMBER']), highest_number) |
231 if base and match.groupdict()['BASE'] != base: | 206 if base and match.groupdict()['BASE'] != base: |
232 raise Exception('Illegal wpr file name ' + wpr_file + | 207 raise Exception('Illegal wpr file name ' + wpr_file + |
233 ', doesn\'t begin with ' + base) | 208 ', doesn\'t begin with ' + base) |
234 base = match.groupdict()['BASE'] | 209 base = match.groupdict()['BASE'] |
235 if not base: | 210 if not base: |
236 # If we're creating a completely new info file, use the base name of the | 211 # If we're creating a completely new info file, use the base name of the |
237 # story set file. | 212 # story set file. |
238 base = os.path.splitext(os.path.basename(self._file_path))[0] | 213 base = os.path.splitext(os.path.basename(self._file_path))[0] |
239 new_filename = '%s_%03d.%s' % (base, highest_number + 1, extension) | 214 new_filename = '%s_%03d.%s' % (base, highest_number + 1, extension) |
240 return new_filename, self._WprFileNameToPath(new_filename) | 215 return new_filename, self._WprFileNameToPath(new_filename) |
241 | 216 |
242 def _SetWprFileForStory(self, story_name, wpr_file, target_platform): | 217 def _SetWprFileForStory(self, story_name, wpr_file, target_platform): |
243 """For modifying the metadata when we're going to record a new archive.""" | 218 """For modifying the metadata when we're going to record a new archive.""" |
244 if story_name not in self._data['archives']: | 219 if story_name not in self._data['archives']: |
245 # If there is no other recording we want the first to be the default | 220 # If there is no other recording we want the first to be the default |
246 # until a new default is recorded. | 221 # until a new default is recorded. |
247 self._data['archives'][story_name] = {_DEFAULT_PLATFORM: wpr_file} | 222 self._data['archives'][story_name] = {_DEFAULT_PLATFORM: wpr_file} |
248 self._data['archives'][story_name][target_platform] = wpr_file | 223 self._data['archives'][story_name][target_platform] = wpr_file |
OLD | NEW |