OLD | NEW |
1 # Copyright 2016 The LUCI Authors. All rights reserved. | 1 # Copyright 2016 The LUCI Authors. All rights reserved. |
2 # Use of this source code is governed under the Apache License, Version 2.0 | 2 # Use of this source code is governed under the Apache License, Version 2.0 |
3 # that can be found in the LICENSE file. | 3 # that can be found in the LICENSE file. |
4 | 4 |
5 import base64 | 5 import base64 |
6 import httplib | 6 import httplib |
7 import json | 7 import json |
8 import logging | 8 import logging |
9 import os | 9 import os |
| 10 import re |
10 import shutil | 11 import shutil |
11 import sys | 12 import sys |
12 import tarfile | 13 import tarfile |
13 import tempfile | 14 import tempfile |
14 | 15 |
15 # Add third party paths. | 16 # Add third party paths. |
16 from . import env | 17 from . import env |
17 from . import requests_ssl | 18 from . import requests_ssl |
18 from . import util | 19 from . import util |
| 20 from . import types |
19 from .requests_ssl import requests | 21 from .requests_ssl import requests |
20 | 22 |
21 import subprocess42 | 23 import subprocess42 |
22 from google.protobuf import text_format | 24 from google.protobuf import text_format |
23 | 25 |
24 from . import package_pb2 | 26 from . import package_pb2 |
25 | 27 |
26 | 28 |
27 class FetchError(Exception): | 29 class FetchError(Exception): |
28 pass | 30 pass |
(...skipping 24 matching lines...) Expand all Loading... |
53 def updates(self, repo, revision, checkout_dir, allow_fetch, | 55 def updates(self, repo, revision, checkout_dir, allow_fetch, |
54 other_revision, paths): | 56 other_revision, paths): |
55 """Returns a list of revisions between |revision| and |other_revision|. | 57 """Returns a list of revisions between |revision| and |other_revision|. |
56 | 58 |
57 Network operations are performed only if |allow_fetch| is True. | 59 Network operations are performed only if |allow_fetch| is True. |
58 | 60 |
59 If |paths| is a non-empty list, the history is scoped just to these paths. | 61 If |paths| is a non-empty list, the history is scoped just to these paths. |
60 """ | 62 """ |
61 raise NotImplementedError() | 63 raise NotImplementedError() |
62 | 64 |
| 65 # This is a simple mapping of |
| 66 # repo -> git_revision -> commit_metadata() |
| 67 # It only holds cache entries for git commits (e.g. sha1 hashes) |
| 68 _GIT_METADATA_CACHE = {} |
| 69 |
| 70 # This matches git commit hashes. |
| 71 _COMMIT_RE = re.compile('^[a-fA-F0-9]{40}$') |
| 72 |
63 def commit_metadata(self, repo, revision, checkout_dir, allow_fetch): | 73 def commit_metadata(self, repo, revision, checkout_dir, allow_fetch): |
| 74 """Cached version of commit_metadata_slow. Implementations should only |
| 75 override commit_metadata_slow and this implementation will call it if |
| 76 there's no cached value for (repo, revision). |
| 77 """ |
| 78 if self._COMMIT_RE.match(revision): |
| 79 cache = self._GIT_METADATA_CACHE.setdefault(repo, {}) |
| 80 if revision not in cache: |
| 81 cache[revision] = types.freeze(self.commit_metadata_slow( |
| 82 repo, revision, checkout_dir, allow_fetch)) |
| 83 return cache[revision] |
| 84 return self.commit_metadata_slow(repo, revision, checkout_dir, allow_fetch) |
| 85 |
| 86 def commit_metadata_slow(self, repo, revision, checkout_dir, allow_fetch): |
64 """Returns a dictionary of metadata about commit |revision|. | 87 """Returns a dictionary of metadata about commit |revision|. |
65 | 88 |
66 The dictionary contains the following keys: author, message. | 89 The dictionary contains the following keys: author, message. |
| 90 |
| 91 This implementation will likely do slow operations (e.g. subprocess |
| 92 invocations, network access, etc.). Users of Backend will always want to use |
| 93 commit_metadata() instead. |
67 """ | 94 """ |
68 raise NotImplementedError() | 95 raise NotImplementedError() |
69 | 96 |
70 | 97 |
71 class UncleanFilesystemError(FetchError): | 98 class UncleanFilesystemError(FetchError): |
72 pass | 99 pass |
73 | 100 |
74 | 101 |
75 class GitError(FetchError): | 102 class GitError(FetchError): |
76 | 103 |
(...skipping 91 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
168 | 195 |
169 args = [ | 196 args = [ |
170 'rev-list', | 197 'rev-list', |
171 '--reverse', | 198 '--reverse', |
172 '%s..%s' % (revision, other_revision), | 199 '%s..%s' % (revision, other_revision), |
173 ] | 200 ] |
174 if paths: | 201 if paths: |
175 args.extend(['--'] + paths) | 202 args.extend(['--'] + paths) |
176 return filter(bool, git(*args).strip().split('\n')) | 203 return filter(bool, git(*args).strip().split('\n')) |
177 | 204 |
178 def commit_metadata(self, repo, revision, checkout_dir, allow_fetch): | 205 def commit_metadata_slow(self, repo, revision, checkout_dir, allow_fetch): |
179 git = self.Git(checkout_dir=checkout_dir) | 206 git = self.Git(checkout_dir=checkout_dir) |
180 return { | 207 return { |
181 'author': git('show', '-s', '--pretty=%aE', revision).strip(), | 208 'author': git('show', '-s', '--pretty=%aE', revision).strip(), |
182 'message': git('show', '-s', '--pretty=%B', revision).strip(), | 209 'message': git('show', '-s', '--pretty=%B', revision).strip(), |
183 } | 210 } |
184 | 211 |
185 | 212 |
186 class GitilesFetchError(FetchError): | 213 class GitilesFetchError(FetchError): |
187 """An HTTP error that occurred during Gitiles fetching.""" | 214 """An HTTP error that occurred during Gitiles fetching.""" |
188 | 215 |
(...skipping 94 matching lines...) Expand 10 before | Expand all | Expand 10 after Loading... |
283 diff_entry['new_path'].startswith(path)): | 310 diff_entry['new_path'].startswith(path)): |
284 matched = True | 311 matched = True |
285 break | 312 break |
286 if matched: | 313 if matched: |
287 break | 314 break |
288 if matched or not paths: | 315 if matched or not paths: |
289 results.append(entry['commit']) | 316 results.append(entry['commit']) |
290 | 317 |
291 return list(reversed(results)) | 318 return list(reversed(results)) |
292 | 319 |
293 def commit_metadata(self, repo, revision, checkout_dir, allow_fetch): | 320 def commit_metadata_slow(self, repo, revision, checkout_dir, allow_fetch): |
294 if not allow_fetch: | 321 if not allow_fetch: |
295 raise FetchNotAllowedError( | 322 raise FetchNotAllowedError( |
296 ('requested commit metadata for %s (%s)from gitiles but fetch not ' | 323 ('requested commit metadata for %s (%s)from gitiles but fetch not ' |
297 'allowed') % (repo, revision)) | 324 'allowed') % (repo, revision)) |
298 rev_json = self._revision_metadata(repo, revision) | 325 rev_json = self._revision_metadata(repo, revision) |
299 return { | 326 return { |
300 'author': rev_json['author']['email'], | 327 'author': rev_json['author']['email'], |
301 'message': rev_json['message'], | 328 'message': rev_json['message'], |
302 } | 329 } |
303 | 330 |
(...skipping 27 matching lines...) Expand all Loading... |
331 logging.info('fetching %s', url) | 358 logging.info('fetching %s', url) |
332 | 359 |
333 resp = requests.get(url) | 360 resp = requests.get(url) |
334 if resp.status_code != httplib.OK: | 361 if resp.status_code != httplib.OK: |
335 raise GitilesFetchError(resp.status_code, resp.text) | 362 raise GitilesFetchError(resp.status_code, resp.text) |
336 | 363 |
337 if not resp.text.startswith(cls._GERRIT_XSRF_HEADER): | 364 if not resp.text.startswith(cls._GERRIT_XSRF_HEADER): |
338 raise GitilesFetchError(resp.status_code, 'Missing XSRF header') | 365 raise GitilesFetchError(resp.status_code, 'Missing XSRF header') |
339 | 366 |
340 return json.loads(resp.text[len(cls._GERRIT_XSRF_HEADER):]) | 367 return json.loads(resp.text[len(cls._GERRIT_XSRF_HEADER):]) |
OLD | NEW |