| OLD | NEW |
| 1 # Copyright 2013 The Chromium Authors. All rights reserved. | 1 # Copyright 2013 The Chromium Authors. All rights reserved. |
| 2 # Use of this source code is governed by a BSD-style license that can be | 2 # Use of this source code is governed by a BSD-style license that can be |
| 3 # found in the LICENSE file. | 3 # found in the LICENSE file. |
| 4 | 4 |
| 5 import json | 5 import json |
| 6 import logging | 6 import logging |
| 7 import os | 7 import os |
| 8 import re | 8 import re |
| 9 import shutil | 9 import shutil |
| 10 import tempfile | 10 import tempfile |
| (...skipping 24 matching lines...) Expand all Loading... |
| 35 self._bucket = bucket | 35 self._bucket = bucket |
| 36 self.temp_target_wpr_file_path = None | 36 self.temp_target_wpr_file_path = None |
| 37 # Ensure directory exists. | 37 # Ensure directory exists. |
| 38 if not os.path.exists(self._base_dir): | 38 if not os.path.exists(self._base_dir): |
| 39 os.makedirs(self._base_dir) | 39 os.makedirs(self._base_dir) |
| 40 | 40 |
| 41 assert data.get('platform_specific', False), ( | 41 assert data.get('platform_specific', False), ( |
| 42 'Detected old version of archive info json file. Please update to new ' | 42 'Detected old version of archive info json file. Please update to new ' |
| 43 'version.') | 43 'version.') |
| 44 | 44 |
| 45 self.is_using_wpr_go_archives = True | |
| 46 self._story_name_to_wpr_file = data['archives'] | 45 self._story_name_to_wpr_file = data['archives'] |
| 47 story_archives = self._data['archives'] | |
| 48 for story in story_archives: | |
| 49 files = story_archives[story] | |
| 50 for f in files: | |
| 51 if files[f].endswith('.wpr'): | |
| 52 self.is_using_wpr_go_archives = False | |
| 53 self.ValidateArchivesFormat(story_archives) | |
| 54 | |
| 55 @staticmethod | |
| 56 def ValidateArchivesFormat(story_archives): | |
| 57 """ Checks that all archives follow either .wpr format or .wprgo format | |
| 58 but not both. | |
| 59 """ | |
| 60 using_wpr = False | |
| 61 using_wpr_go = False | |
| 62 for story in story_archives: | |
| 63 files = story_archives[story] | |
| 64 for f in files: | |
| 65 if files[f].endswith('.wprgo'): | |
| 66 using_wpr_go = True | |
| 67 elif files[f].endswith('.wpr'): | |
| 68 using_wpr = True | |
| 69 assert not(using_wpr and using_wpr_go), ( | |
| 70 'Detected both .wprgo or .wpr archive format.') | |
| 71 | 46 |
| 72 @classmethod | 47 @classmethod |
| 73 def FromFile(cls, file_path, bucket): | 48 def FromFile(cls, file_path, bucket): |
| 74 """ Generates an archive_info instance with the given json file. """ | 49 """ Generates an archive_info instance with the given json file. """ |
| 75 if os.path.exists(file_path): | 50 if os.path.exists(file_path): |
| 76 with open(file_path, 'r') as f: | 51 with open(file_path, 'r') as f: |
| 77 data = json.load(f) | 52 data = json.load(f) |
| 78 return cls(file_path, data, bucket) | 53 return cls(file_path, data, bucket) |
| 79 return cls(file_path, {'archives': {}, 'platform_specific': True}, bucket) | 54 return cls(file_path, {'archives': {}, 'platform_specific': True}, bucket) |
| 80 | 55 |
| (...skipping 10 matching lines...) Expand all Loading... |
| 91 exists. | 66 exists. |
| 92 """ | 67 """ |
| 93 logging.info('Downloading WPR archives. This can take a long time.') | 68 logging.info('Downloading WPR archives. This can take a long time.') |
| 94 start_time = time.time() | 69 start_time = time.time() |
| 95 # If no target platform is set, download all platforms. | 70 # If no target platform is set, download all platforms. |
| 96 if target_platforms is None: | 71 if target_platforms is None: |
| 97 target_platforms = _ALL_PLATFORMS | 72 target_platforms = _ALL_PLATFORMS |
| 98 else: | 73 else: |
| 99 assert isinstance(target_platforms, list), 'Must pass platforms as a list' | 74 assert isinstance(target_platforms, list), 'Must pass platforms as a list' |
| 100 target_platforms = target_platforms + [_DEFAULT_PLATFORM] | 75 target_platforms = target_platforms + [_DEFAULT_PLATFORM] |
| 101 # Download all .wpr or .wprgo files. | 76 # Download all .wprgo files. |
| 102 if not self._bucket: | 77 if not self._bucket: |
| 103 logging.warning('Story set in %s has no bucket specified, and ' | 78 logging.warning('Story set in %s has no bucket specified, and ' |
| 104 'cannot be downloaded from cloud_storage.', ) | 79 'cannot be downloaded from cloud_storage.', ) |
| 105 return | 80 return |
| 106 assert 'archives' in self._data, ("Invalid data format in %s. 'archives' " | 81 assert 'archives' in self._data, ("Invalid data format in %s. 'archives' " |
| 107 "field is needed" % self._file_path) | 82 "field is needed" % self._file_path) |
| 108 | 83 |
| 109 def download_if_needed(path): | 84 def download_if_needed(path): |
| 110 try: | 85 try: |
| 111 cloud_storage.GetIfChanged(path, self._bucket) | 86 cloud_storage.GetIfChanged(path, self._bucket) |
| (...skipping 97 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
| 209 def _WprFileNameToPath(self, wpr_file): | 184 def _WprFileNameToPath(self, wpr_file): |
| 210 return os.path.abspath(os.path.join(self._base_dir, wpr_file)) | 185 return os.path.abspath(os.path.join(self._base_dir, wpr_file)) |
| 211 | 186 |
| 212 def _NextWprFileName(self): | 187 def _NextWprFileName(self): |
| 213 """Creates a new file name for a wpr archive file.""" | 188 """Creates a new file name for a wpr archive file.""" |
| 214 # The names are of the format "some_thing_number.wpr" or | 189 # The names are of the format "some_thing_number.wpr" or |
| 215 # "some_thing_number.wprgo". Read the numbers. | 190 # "some_thing_number.wprgo". Read the numbers. |
| 216 highest_number = -1 | 191 highest_number = -1 |
| 217 base = None | 192 base = None |
| 218 wpr_files = [] | 193 wpr_files = [] |
| 219 extension = 'wprgo' if self.is_using_wpr_go_archives else 'wpr' | 194 extension = 'wprgo' |
| 220 for story in self._data['archives']: | 195 for story in self._data['archives']: |
| 221 for p in self._data['archives'][story]: | 196 for p in self._data['archives'][story]: |
| 222 wpr_files.append(self._data['archives'][story][p]) | 197 wpr_files.append(self._data['archives'][story][p]) |
| 223 | 198 |
| 224 for wpr_file in wpr_files: | 199 for wpr_file in wpr_files: |
| 225 pattern = r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+).{extension}'.format( | 200 pattern = r'(?P<BASE>.*)_(?P<NUMBER>[0-9]+).{extension}'.format( |
| 226 extension=extension) | 201 extension=extension) |
| 227 match = re.match(pattern, wpr_file) | 202 match = re.match(pattern, wpr_file) |
| 228 if not match: | 203 if not match: |
| 229 raise Exception('Illegal wpr file name ' + wpr_file) | 204 raise Exception('Illegal wpr file name ' + wpr_file) |
| 230 highest_number = max(int(match.groupdict()['NUMBER']), highest_number) | 205 highest_number = max(int(match.groupdict()['NUMBER']), highest_number) |
| 231 if base and match.groupdict()['BASE'] != base: | 206 if base and match.groupdict()['BASE'] != base: |
| 232 raise Exception('Illegal wpr file name ' + wpr_file + | 207 raise Exception('Illegal wpr file name ' + wpr_file + |
| 233 ', doesn\'t begin with ' + base) | 208 ', doesn\'t begin with ' + base) |
| 234 base = match.groupdict()['BASE'] | 209 base = match.groupdict()['BASE'] |
| 235 if not base: | 210 if not base: |
| 236 # If we're creating a completely new info file, use the base name of the | 211 # If we're creating a completely new info file, use the base name of the |
| 237 # story set file. | 212 # story set file. |
| 238 base = os.path.splitext(os.path.basename(self._file_path))[0] | 213 base = os.path.splitext(os.path.basename(self._file_path))[0] |
| 239 new_filename = '%s_%03d.%s' % (base, highest_number + 1, extension) | 214 new_filename = '%s_%03d.%s' % (base, highest_number + 1, extension) |
| 240 return new_filename, self._WprFileNameToPath(new_filename) | 215 return new_filename, self._WprFileNameToPath(new_filename) |
| 241 | 216 |
| 242 def _SetWprFileForStory(self, story_name, wpr_file, target_platform): | 217 def _SetWprFileForStory(self, story_name, wpr_file, target_platform): |
| 243 """For modifying the metadata when we're going to record a new archive.""" | 218 """For modifying the metadata when we're going to record a new archive.""" |
| 244 if story_name not in self._data['archives']: | 219 if story_name not in self._data['archives']: |
| 245 # If there is no other recording we want the first to be the default | 220 # If there is no other recording we want the first to be the default |
| 246 # until a new default is recorded. | 221 # until a new default is recorded. |
| 247 self._data['archives'][story_name] = {_DEFAULT_PLATFORM: wpr_file} | 222 self._data['archives'][story_name] = {_DEFAULT_PLATFORM: wpr_file} |
| 248 self._data['archives'][story_name][target_platform] = wpr_file | 223 self._data['archives'][story_name][target_platform] = wpr_file |
| OLD | NEW |