Coverage for dtuf/__init__.py : 92%
Hot-keys on this page
r m x p toggle line displays
j k next/prev highlighted chunk
0 (zero) top of page
1 (one) first highlighted chunk
1"""
2Docker registry bindings for The Update Framework
3"""
5# pylint: disable=superfluous-parens, wrong-import-order
7try:
8 import urllib.parse as urlparse
9except ImportError:
10 # pylint: disable=import-error
11 import urlparse
13import json
14import threading
15import hashlib
16import re
17from os import path, getcwd, remove, makedirs, listdir
18from datetime import datetime, timedelta
19import tempfile
20from decorator import decorator
21import fasteners
22from dxf import DXFBase, DXF, hash_file, hash_bytes, split_digest
23import dxf.exceptions
24from dtuf import exceptions
25import iso8601
26import securesystemslib.util
27import securesystemslib.settings
28securesystemslib.settings.HASH_ALGORITHMS = ['sha256']
30class _TufResponse(object):
31 def __init__(self, url, request_kwargs):
32 self._url = url
33 self._request_kwargs = request_kwargs
34 self._it = None
35 self._count = 0
37 def __enter__(self):
38 return self
40 def raise_for_status(self):
41 #pylint: disable=attribute-defined-outside-init,protected-access
42 import tuf.settings
43 target = urlparse.urlparse(self._url).path.split('/')[-1]
44 pos = _skip_consistent_target_digest(target)
45 _updater_dxf._request_kwargs = self._request_kwargs
46 try:
47 if pos == 0:
48 self._dgst = _updater_dxf.get_alias(target)[0]
49 else:
50 self._dgst = 'sha256:' + target[0:pos-1]
51 self._it, self._size = _updater_dxf.pull_blob(
52 self._dgst, size=True, chunk_size=tuf.settings.CHUNK_SIZE)
53 self._it = self._it.__iter__()
54 self._end = object()
55 if _updater_progress:
56 _updater_progress(self._dgst, b'', self._size)
57 finally:
58 _updater_dxf._request_kwargs = {}
60 @property
61 def raw(self):
62 return self
64 def read(self, read_amount): #pylint: disable=unused-argument
65 chunk = next(self._it, self._end)
66 if chunk is self._end:
67 return b''
68 if _updater_progress:
69 _updater_progress(self._dgst, chunk, self._size)
70 self._count += len(chunk)
71 return chunk
73 def close(self):
74 # give dxf a chance to check the digest
75 if (self._it is not None) and (self._count == self._size):
76 next(self._it, self._end)
78 def __exit__(self, *args):
79 self.close()
81 def __str__(self):
82 return "dtuf connection to %s" % self._url
84class _TufSession(object): #pylint: disable=too-few-public-methods
85 def get(self, url, **kwargs): #pylint: disable=no-self-use
86 return _TufResponse(url, kwargs)
88class _TufSessions(object): #pylint: disable=too-few-public-methods
89 def get(self, session_index): #pylint: disable=unused-argument,no-self-use
90 return _TufSession()
92_tuf_sessions = _TufSessions()
94class _UpdaterDXF(DXF):
95 def __init__(self, *args, **kwargs):
96 super(_UpdaterDXF, self).__init__(*args, **kwargs)
97 self._request_kwargs = {}
99 def _base_request(self, method, path, **kwargs): #pylint: disable=redefined-outer-name
100 request_kwargs = {}
101 request_kwargs.update(self._request_kwargs)
102 request_kwargs.update(kwargs)
103 return super(_UpdaterDXF, self)._base_request(method, path, **request_kwargs)
105def _is_metadata_file(target):
106 return target.endswith('root.json') or \
107 target.endswith('targets.json') or \
108 target.endswith('snapshot.json') or \
109 target.endswith('timestamp.json')
111_tuf_lock = threading.Lock()
112_updater_dxf = None
113_updater_progress = None
115def _tuf_clear():
116 import tuf.keydb
117 import tuf.roledb
118 import tuf.settings
119 # pylint: disable=global-statement
120 global _updater_dxf, _updater_progress
121 _updater_dxf = None
122 _updater_progress = None
123 tuf.keydb.clear_keydb()
124 tuf.roledb.clear_roledb()
125 tuf.settings.repositories_directory = None
127def _locked(lock, setup, f, self, *args, **kwargs):
128 with _tuf_lock:
129 with lock:
130 _tuf_clear()
131 try:
132 if setup:
133 setup()
134 return f(self, *args, **kwargs)
135 finally:
136 _tuf_clear()
138@decorator
139def _master_repo_locked(f, self, *args, **kwargs):
140 # pylint: disable=protected-access
141 return _locked(self._master_repo_lock, None, f, self, *args, **kwargs)
143@decorator
144def _copy_repo_locked(f, self, *args, **kwargs):
145 # pylint: disable=global-statement,protected-access
146 def setup():
147 import tuf.settings
148 import tuf.download
149 # pylint: disable=protected-access
150 if hasattr(tuf.download, '_sessions'): 150 ↛ 153line 150 didn't jump to line 153, because the condition on line 150 was never false
151 tuf.download._sessions = _tuf_sessions
152 global _updater_dxf
153 _updater_dxf = self._dxf
154 tuf.settings.repositories_directory = self._copy_dir
155 return _locked(self._copy_repo_lock, setup, f, self, *args, **kwargs)
157_consistent_prefix_re = re.compile(r'[a-f0-9]{' +
158 str(hashlib.sha256().digest_size * 2) +
159 r'}\.')
161def _skip_consistent_target_digest(basename):
162 match = _consistent_prefix_re.match(basename)
163 return match.end() if match else 0
165def _strip_consistent_target_digest(filename):
166 dirname, basename = path.split(filename)
167 return path.join(dirname,
168 basename[_skip_consistent_target_digest(basename):])
170def _remove_keys(metadata):
171 import tuf.keydb
172 import tuf.roledb
173 for keyid in tuf.roledb.get_roleinfo(metadata.rolename)['keyids']:
174 metadata.remove_verification_key(tuf.keydb.get_key(keyid))
176def _write_with_progress(it, dgst, size, out, progress):
177 if progress:
178 progress(dgst, b'', size)
179 for chunk in it:
180 if progress:
181 progress(dgst, chunk, size)
182 out.write(chunk)
184class DTufBase(object):
185 """
186 Class for communicating with a Docker v2 registry.
187 Contains only operations which aren't related to pushing and pulling data
188 to repositories in the registry using
189 `The Update Framework <https://github.com/theupdateframework/tuf>`_.
191 Can act as a context manager. For each context entered, a new
192 `requests.Session <http://docs.python-requests.org/en/latest/user/advanced/#session-objects>`_
193 is obtained. Connections to the same host are shared by the session.
194 When the context exits, all the session's connections are closed.
196 If you don't use :class:`DTufBase` as a context manager, each request
197 uses an ephemeral session. If you don't read all the data from an iterator
198 returned by :meth:`DTufCopy.pull_target` then the underlying connection
199 won't be closed until Python garbage collects the iterator.
200 """
201 def __init__(self, host, auth=None, insecure=False, auth_host=None):
202 """
203 :param host: Host name of registry. Can contain port numbers. e.g. ``registry-1.docker.io``, ``localhost:5000``.
204 :type host: str
206 :param auth: Authentication function to be called whenever authentication to the registry is required. Receives the :class:`DTufBase` object and a HTTP response object. It should call :meth:`authenticate` on ``dtuf_obj`` with a username, password and ``response`` before it returns.
207 :type auth: function(dtuf_obj, response)
209 :param insecure: Use HTTP instead of HTTPS (which is the default) when connecting to the registry.
210 :type insecure: bool
212 :param auth_host: Host to use for token authentication. If set, overrides host returned by then registry.
213 :type auth_host: str
214 """
215 self._dxf = DXFBase(host, self._wrap_auth(auth), insecure, auth_host)
217 def _wrap_auth(self, auth=None):
218 return lambda dxf_obj, response: auth(self, response) if auth else None
220 @property
221 def token(self):
222 """
223 str: Authentication token. This will be obtained automatically when
224 you call :meth:`authenticate`. If you've obtained a token
225 previously, you can also set it but be aware tokens expire quickly.
226 """
227 return self._dxf.token
229 @token.setter
230 def token(self, value):
231 self._dxf.token = value
233 def authenticate(self,
234 username=None, password=None,
235 actions=None, response=None):
236 """
237 Authenticate to the registry, using a username and password if supplied,
238 otherwise as the anonymous user.
240 :param username: User name to authenticate as.
241 :type username: str
243 :param password: User's password.
244 :type password: str
246 :param actions: If you know which types of operation you need to make on the registry, specify them here. Valid actions are ``pull``, ``push`` and ``*``.
247 :type actions: list
249 :param response: When the ``auth`` function you passed to :class:`DTufBase`'s constructor is called, it is passed a HTTP response object. Pass it back to :meth:`authenticate` to have it automatically detect which actions are required.
250 :type response: requests.Response
252 :rtype: str
253 :returns: Authentication token, if the registry supports bearer tokens. Otherwise ``None``, and HTTP Basic auth is used.
254 """
255 return self._dxf.authenticate(username, password, actions, response)
257 def list_repos(self):
258 """
259 List all repositories in the registry.
261 :rtype: list
262 :returns: List of repository names.
263 """
264 return self._dxf.list_repos()
266 def __enter__(self):
267 self._dxf.__enter__()
268 return self
270 def __exit__(self, *args):
271 return self._dxf.__exit__(*args)
273class _DTufCommon(DTufBase):
274 # pylint: disable=too-many-arguments,super-init-not-called
275 def __init__(self, host, repo, repos_root=None,
276 auth=None, insecure=False, auth_host=None):
277 self._dxf = _UpdaterDXF(host, repo, self._wrap_auth(auth), insecure, auth_host)
278 self._repo_root = path.join(repos_root if repos_root else path.join(getcwd(), 'dtuf_repos'), repo)
279 import tuf.settings
280 tuf.settings.FILE_HASH_ALGORITHMS = ['sha256']
282# pylint: disable=too-many-instance-attributes
283class DTufMaster(_DTufCommon):
284 """
285 Class for creating, updating and publishing data to repositories in a
286 Docker registry using
287 `The Update Framework <https://github.com/theupdateframework/tuf>`_ (TUF).
288 """
289 # pylint: disable=too-many-arguments,too-many-locals
290 def __init__(self, host, repo, repos_root=None,
291 auth=None, insecure=False, auth_host=None,
292 root_lifetime=None, targets_lifetime=None,
293 snapshot_lifetime=None, timestamp_lifetime=None):
294 """
295 :param host: Host name of registry. Can contain port numbers. e.g. ``registry-1.docker.io``, ``localhost:5000``.
296 :type host: str
298 :param repo: Name of the repository to access on the registry. Typically this is of the form ``username/reponame`` but for your own registries you don't actually have to stick to that. The repository is used to store data and TUF metadata describing the data.
299 :type repo: str
301 :param repos_root: Directory under which to store TUF metadata. Note that the value of ``repo`` and the literal string ``master`` are appended to this directory name before storing the metadata. Defaults to ``dtuf_repos`` in the current working directory.
302 :type repos_root: str
304 :param auth: Authentication function to be called whenever authentication to the registry is required. Receives the :class:`DTufBase` object and a HTTP response object. It should call :meth:`DTufBase.authenticate` on ``dtuf_obj`` with a username, password and ``response`` before it returns.
305 :type auth: function(dtuf_obj, response)
307 :param insecure: Use HTTP instead of HTTPS (which is the default) when connecting to the registry.
308 :type insecure: bool
310 :param auth_host: Host to use for token authentication. If set, overrides host returned by then registry.
311 :type auth_host: str
313 :param root_lifetime: Lifetime of the TUF `root metadata <https://github.com/theupdateframework/tuf/blob/develop/docs/tuf-spec.txt#L235>`_. After this time expires, you'll need to call :meth:`reset_keys` and :meth:`push_metadata` to re-sign the metadata. Defaults to ``tuf.repository_tool.ROOT_EXPIRATION`` (1 year).
314 :type root_lifetime: datetime.timedelta
316 :param targets_lifetime: Lifetime of the TUF `targets metadata <https://github.com/theupdateframework/tuf/blob/develop/docs/tuf-spec.txt#L246>`_. After this time expires, you'll need to call :meth:`push_metadata` to re-sign the metadata. Defaults to ``tuf.repository_tool.TARGETS_EXPIRATION`` (3 months).
317 :type targets_lifetime: datetime.timedelta
319 :param snapshot_lifetime: Lifetime of the TUF `snapshot metadata <https://github.com/theupdateframework/tuf/blob/develop/docs/tuf-spec.txt#L268>`_. After this time expires, you'll need to call :meth:`push_metadata` to re-sign the metadata. Defaults to ``tuf.repository_tool.SNAPSHOT_EXPIRATION`` (1 week).
320 :type snapshot_lifetime: datetime.timedelta
322 :param timestamp_lifetime: Lifetime of the TUF `timestamp metadata <https://github.com/theupdateframework/tuf/blob/develop/docs/tuf-spec.txt#L276>`_. After this time expires, you'll need to call :meth:`push_metadata` to re-sign the metadata. Defaults to ``tuf.repository_tool.TIMESTAMP_EXPIRATION`` (1 day).
323 :type timestamp_lifetime: datetime.timedelta
324 """
325 super(DTufMaster, self).__init__(host, repo, repos_root,
326 auth, insecure, auth_host)
327 self._master_dir = path.join(self._repo_root, 'master')
328 self._master_repo_lock = fasteners.process_lock.InterProcessLock(
329 path.join(self._master_dir, 'lock'))
330 self._keys_dir = path.join(self._master_dir, 'keys')
331 self._root_key_file = path.join(self._keys_dir, 'root_key')
332 self._targets_key_file = path.join(self._keys_dir, 'targets_key')
333 self._snapshot_key_file = path.join(self._keys_dir, 'snapshot_key')
334 self._timestamp_key_file = path.join(self._keys_dir, 'timestamp_key')
335 self._master_repo_dir = path.join(self._master_dir, 'repository')
336 self._master_targets_dir = path.join(self._master_repo_dir, 'targets')
337 self._master_staged_dir = path.join(self._master_repo_dir, 'metadata.staged')
338 from tuf.repository_tool import ROOT_EXPIRATION, \
339 TARGETS_EXPIRATION, \
340 SNAPSHOT_EXPIRATION, \
341 TIMESTAMP_EXPIRATION
342 self._root_lifetime = timedelta(seconds=ROOT_EXPIRATION) \
343 if root_lifetime is None else root_lifetime
344 self._targets_lifetime = timedelta(seconds=TARGETS_EXPIRATION) \
345 if targets_lifetime is None else targets_lifetime
346 self._snapshot_lifetime = timedelta(seconds=SNAPSHOT_EXPIRATION) \
347 if snapshot_lifetime is None else snapshot_lifetime
348 self._timestamp_lifetime = timedelta(seconds=TIMESTAMP_EXPIRATION) \
349 if timestamp_lifetime is None else timestamp_lifetime
351 @_master_repo_locked
352 def create_root_key(self, password=None):
353 """
354 Create root keypair for the repository.
356 The private key is written to ``<repos_root>/<repo>/master/keys/root_key`` and can be moved offline once you've called :meth:`create_metadata`. You'll need it again if you call :meth:`reset_keys` when the root metadata expires.
358 The public key is written to ``<repos_root>/<repo>/master/keys/root_key.pub`` and can be given to others for use when retrieving a copy of the repository metadata with :meth:`DTufCopy.pull_metadata`.
360 :param password: Password to use for encrypting the private key. You'll be prompted for one if you don't supply it.
361 :type password: str
362 """
363 from tuf.repository_tool import generate_and_write_rsa_keypair, \
364 generate_and_write_rsa_keypair_with_prompt
365 if password is None: 365 ↛ 366line 365 didn't jump to line 366, because the condition on line 365 was never true
366 print('generating root key...')
367 generate_and_write_rsa_keypair_with_prompt(filepath=self._root_key_file)
368 else:
369 generate_and_write_rsa_keypair(password, filepath=self._root_key_file)
371 @_master_repo_locked
372 def create_metadata_keys(self,
373 targets_key_password=None,
374 snapshot_key_password=None,
375 timestamp_key_password=None):
376 """
377 Create TUF metadata keypairs for the repository.
379 The keys are written to the ``<repos_root>/<repo>/master/keys`` directory. The public keys have a ``.pub`` extension.
381 You can move the private keys offline once you've called :meth:`create_metadata` but you'll need them again when you call :meth:`push_metadata` to publish the repository.
383 You don't need to give out the metadata public keys since they're published on the repository.
385 :param targets_key_password: Password to use for encrypting the TUF targets private key. You'll be prompted for one if you don't supply it.
386 :type password: str
388 :param snapshot_key_password: Password to use for encrypting the TUF snapshot private key. You'll be prompted for one if you don't supply it.
389 :type password: str
391 :param timestamp_key_password: Password to use for encrypting the TUF timestamp private key. You'll be prompted for one if you don't supply it.
392 :type password: str
393 """
394 from tuf.repository_tool import generate_and_write_rsa_keypair, \
395 generate_and_write_rsa_keypair_with_prompt
397 if targets_key_password is None: 397 ↛ 398line 397 didn't jump to line 398, because the condition on line 397 was never true
398 print('generating targets key...')
399 generate_and_write_rsa_keypair_with_prompt(filepath=self._targets_key_file)
400 else:
401 generate_and_write_rsa_keypair(targets_key_password, filepath=self._targets_key_file)
403 if snapshot_key_password is None: 403 ↛ 404line 403 didn't jump to line 404, because the condition on line 403 was never true
404 print('generating snapshot key...')
405 generate_and_write_rsa_keypair_with_prompt(filepath=self._snapshot_key_file)
406 else:
407 generate_and_write_rsa_keypair(snapshot_key_password, filepath=self._snapshot_key_file)
409 if timestamp_key_password is None: 409 ↛ 410line 409 didn't jump to line 410, because the condition on line 409 was never true
410 print('generating timestamp key...')
411 generate_and_write_rsa_keypair_with_prompt(filepath=self._timestamp_key_file)
412 else:
413 generate_and_write_rsa_keypair(timestamp_key_password, filepath=self._timestamp_key_file)
415 # pylint: disable=too-many-locals
416 def _add_metadata(self,
417 repository,
418 root_key_password=None,
419 targets_key_password=None,
420 snapshot_key_password=None,
421 timestamp_key_password=None):
422 from tuf.repository_tool import import_rsa_publickey_from_file
423 from tuf.repository_lib import import_rsa_privatekey_from_file
425 # Add root key to repository
426 public_root_key = import_rsa_publickey_from_file(
427 self._root_key_file + '.pub')
428 if root_key_password is None: 428 ↛ 429line 428 didn't jump to line 429, because the condition on line 428 was never true
429 print('importing root key...')
430 private_root_key = import_rsa_privatekey_from_file(
431 self._root_key_file,
432 root_key_password)
433 repository.root.add_verification_key(public_root_key)
434 repository.root.load_signing_key(private_root_key)
435 repository.root.expiration = datetime.utcnow() + self._root_lifetime
437 # Add targets key to repository
438 public_targets_key = import_rsa_publickey_from_file(
439 self._targets_key_file + '.pub')
440 if targets_key_password is None: 440 ↛ 441line 440 didn't jump to line 441, because the condition on line 440 was never true
441 print('importing targets key...')
442 private_targets_key = import_rsa_privatekey_from_file(
443 self._targets_key_file,
444 targets_key_password)
445 repository.targets.add_verification_key(public_targets_key)
446 repository.targets.load_signing_key(private_targets_key)
448 # Add snapshot key to repository
449 public_snapshot_key = import_rsa_publickey_from_file(
450 self._snapshot_key_file + '.pub')
451 if snapshot_key_password is None: 451 ↛ 452line 451 didn't jump to line 452, because the condition on line 451 was never true
452 print('importing snapshot key...')
453 private_snapshot_key = import_rsa_privatekey_from_file(
454 self._snapshot_key_file,
455 snapshot_key_password)
456 repository.snapshot.add_verification_key(public_snapshot_key)
457 repository.snapshot.load_signing_key(private_snapshot_key)
459 # Add timestamp key to repository
460 public_timestamp_key = import_rsa_publickey_from_file(
461 self._timestamp_key_file + '.pub')
462 if timestamp_key_password is None: 462 ↛ 463line 462 didn't jump to line 463, because the condition on line 462 was never true
463 print('importing timestamp key...')
464 private_timestamp_key = import_rsa_privatekey_from_file(
465 self._timestamp_key_file,
466 timestamp_key_password)
467 repository.timestamp.add_verification_key(public_timestamp_key)
468 repository.timestamp.load_signing_key(private_timestamp_key)
470 # Write out metadata
471 repository.writeall(consistent_snapshot=True)
473 @_master_repo_locked
474 def create_metadata(self,
475 root_key_password=None,
476 targets_key_password=None,
477 snapshot_key_password=None,
478 timestamp_key_password=None):
479 """
480 Create and sign the TUF metadata for the repository.
482 You only need to call this once for each repository, and the
483 repository's root and metadata private keys must be available.
485 :param root_key_password: Password to use for decrypting the TUF root private key. You'll be prompted for one if you don't supply it.
486 :type password: str
488 :param targets_key_password: Password to use for decrypting the TUF targets private key. You'll be prompted for one if you don't supply it.
489 :type password: str
491 :param snapshot_key_password: Password to use for decrypting the TUF snapshot private key. You'll be prompted for one if you don't supply it.
492 :type password: str
494 :param timestamp_key_password: Password to use for decrypting the TUF timestamp private key. You'll be prompted for one if you don't supply it.
495 :type password: str
496 """
497 from tuf.repository_tool import create_new_repository
498 # Create repository object and add metadata to it
499 self._add_metadata(create_new_repository(self._master_repo_dir),
500 root_key_password,
501 targets_key_password,
502 snapshot_key_password,
503 timestamp_key_password)
505 @_master_repo_locked
506 def reset_keys(self,
507 root_key_password=None,
508 targets_key_password=None,
509 snapshot_key_password=None,
510 timestamp_key_password=None):
511 """
512 Re-sign the TUF metadata for the repository.
514 Call this if you've generated new root or metadata keys (because one
515 of the keys has been compromised, for example) but you don't want to
516 delete the repository and start again.
518 :param root_key_password: Password to use for decrypting the TUF root private key. You'll be prompted for one if you don't supply it.
519 :type password: str
521 :param targets_key_password: Password to use for decrypting the TUF targets private key. You'll be prompted for one if you don't supply it.
522 :type password: str
524 :param snapshot_key_password: Password to use for decrypting the TUF snapshot private key. You'll be prompted for one if you don't supply it.
525 :type password: str
527 :param timestamp_key_password: Password to use for decrypting the TUF timestamp private key. You'll be prompted for one if you don't supply it.
528 :type password: str
529 """
530 from tuf.repository_tool import load_repository
531 # Load repository object
532 repository = load_repository(self._master_repo_dir)
533 # pylint: disable=no-member
534 # Remove keys
535 _remove_keys(repository.root)
536 _remove_keys(repository.targets)
537 _remove_keys(repository.snapshot)
538 _remove_keys(repository.timestamp)
539 # Add metadata to repository (adds keys)
540 self._add_metadata(repository,
541 root_key_password,
542 targets_key_password,
543 snapshot_key_password,
544 timestamp_key_password)
546 @_master_repo_locked
547 def push_target(self, target, *filename_or_target_list, **kwargs):
548 """
549 Upload data to the repository and update the local TUF metadata.
551 The metadata isn't uploaded until you call :meth:`push_metadata`.
553 The data is given a name (known as the ``target``) and can come from a
554 list of files or existing target names.
556 :param target: Name to give the data.
557 :type target: str
559 :param filename_or_target_list: List of data to upload. Each item is either a filename or an existing target name. Existing target names should be prepended with ``@`` in order to distinguish them from filenames.
560 :type filename_or_target_list: list
562 :param kwargs: Contains an optional ``progress`` member which is a function to call as the upload progresses. The function will be called with the hash of the content of the file currently being uploaded, the blob just read from the file and the total size of the file.
563 :type kwargs: dict({'progress': function(dgst, chunk, total)})
564 """
565 progress = kwargs.get('progress')
566 if _is_metadata_file(target) or \
567 _skip_consistent_target_digest(target) != 0:
568 raise exceptions.DTufReservedTargetError(target)
569 dgsts = []
570 for filename_or_target in filename_or_target_list:
571 if filename_or_target.startswith('@'):
572 with open(path.join(self._master_targets_dir,
573 filename_or_target[1:]), 'rb') as f:
574 manifest = f.read().decode('utf-8')
575 dgsts += self._dxf.get_alias(manifest=manifest)
576 else:
577 dgsts.append(self._dxf.push_blob(filename_or_target, progress))
578 manifest = self._dxf.make_manifest(*dgsts)
579 manifest_filename = path.join(self._master_targets_dir, target)
580 with open(manifest_filename, 'wb') as f:
581 f.write(manifest.encode('utf-8'))
582 self._dxf.push_blob(manifest_filename, progress)
584 @_master_repo_locked
585 def del_target(self, target):
586 """
587 Delete a target (data) from the repository and update the local TUF
588 metadata.
590 The metadata isn't updated on the registry until you call
591 :meth:`push_metadata`.
593 Note that the registry doesn't support deletes yet so expect an error.
595 :param target: The name you gave to the data when it was uploaded using :meth:`push_target`.
596 :type target: str
597 """
598 from tuf.repository_tool import Repository
599 # read target manifest
600 manifest_filename = path.join(self._master_targets_dir, target)
601 with open(manifest_filename, 'rb') as f:
602 manifest = f.read()
603 manifest_dgst = hash_bytes(manifest)
604 # remove target manifest
605 remove(manifest_filename)
606 # remove consistent snapshot links for target
607 for f in Repository.get_filepaths_in_directory(self._master_targets_dir):
608 _, basename = path.split(f)
609 if basename[_skip_consistent_target_digest(basename):] == target:
610 remove(f)
611 # delete blobs manifest points to
612 for dgst in self._dxf.get_alias(manifest=manifest.decode('utf-8')): 612 ↛ 615line 612 didn't jump to line 615, because the loop on line 612 didn't complete
613 self._dxf.del_blob(dgst)
614 # delete manifest blob
615 self._dxf.del_blob(manifest_dgst)
617 # pylint: disable=too-many-locals
618 @_master_repo_locked
619 def push_metadata(self,
620 targets_key_password=None,
621 snapshot_key_password=None,
622 timestamp_key_password=None,
623 progress=None):
624 """
625 Upload local TUF metadata to the repository.
627 The TUF metadata consists of a list of targets (which were uploaded by
628 :meth:`push_target`), a snapshot of the state of the metadata (list of
629 hashes), a timestamp and a list of public keys.
631 This function signs the metadata except for the list of public keys,
632 so you'll need to supply the password to the respective private keys.
634 The list of public keys was signed (along with the rest of the metadata)
635 with the root private key when you called :meth:`create_metadata`
636 (or :meth:`reset_keys`).
638 :param targets_key_password: Password to use for decrypting the TUF targets private key. You'll be prompted for one if you don't supply it.
639 :type password: str
641 :param snapshot_key_password: Password to use for decrypting the TUF snapshot private key. You'll be prompted for one if you don't supply it.
642 :type password: str
644 :param timestamp_key_password: Password to use for decrypting the TUF timestamp private key. You'll be prompted for one if you don't supply it.
645 :type password: str
647 :param progress: Optional function to call as the upload progresses. The function will be called with the hash of the content of the file currently being uploaded, the blob just read from the file and the total size of the file.
648 :type progress: function(dgst, chunk, total)
649 """
650 from tuf.repository_tool import load_repository, \
651 Repository, \
652 import_rsa_privatekey_from_file
653 # Load repository object
654 repository = load_repository(self._master_repo_dir)
655 # pylint: disable=no-member
657 # Update targets
658 repository.targets.clear_targets()
659 repository.targets.add_targets([
660 path.relpath(_strip_consistent_target_digest(f), self._master_targets_dir)
661 for f in Repository.get_filepaths_in_directory(self._master_targets_dir)])
663 # Update expirations
664 repository.targets.expiration = datetime.utcnow() + self._targets_lifetime
665 repository.snapshot.expiration = datetime.utcnow() + self._snapshot_lifetime
666 repository.timestamp.expiration = datetime.utcnow() + self._timestamp_lifetime
668 # Load targets key
669 if targets_key_password is None: 669 ↛ 670line 669 didn't jump to line 670, because the condition on line 669 was never true
670 print('importing targets key...')
671 private_targets_key = import_rsa_privatekey_from_file(
672 self._targets_key_file,
673 targets_key_password)
674 repository.targets.load_signing_key(private_targets_key)
676 # Load snapshot key
677 if snapshot_key_password is None: 677 ↛ 678line 677 didn't jump to line 678, because the condition on line 677 was never true
678 print('importing snapshot key...')
679 private_snapshot_key = import_rsa_privatekey_from_file(
680 self._snapshot_key_file,
681 snapshot_key_password)
682 repository.snapshot.load_signing_key(private_snapshot_key)
684 # Load timestamp key
685 if timestamp_key_password is None: 685 ↛ 686line 685 didn't jump to line 686, because the condition on line 685 was never true
686 print('importing timestamp key...')
687 private_timestamp_key = import_rsa_privatekey_from_file(
688 self._timestamp_key_file,
689 timestamp_key_password)
690 repository.timestamp.load_signing_key(private_timestamp_key)
692 # Update metadata
693 repository.write('targets', consistent_snapshot=True)
694 repository.write('snapshot', consistent_snapshot=True)
695 repository.write('timestamp', consistent_snapshot=True)
697 # Upload metadata files
698 for f in ['root.json',
699 'timestamp.json',
700 '{}.root.json'.format(repository.root.version),
701 '{}.timestamp.json'.format(repository.timestamp.version),
702 '{}.snapshot.json'.format(repository.snapshot.version),
703 '{}.targets.json'.format(repository.targets.version)]:
704 dgst = self._dxf.push_blob(path.join(self._master_staged_dir, f),
705 progress)
706 self._dxf.set_alias(f, dgst)
708 @_master_repo_locked
709 def list_targets(self):
710 """
711 Return the names of all the targets defined in the local TUF metadata.
713 :returns: List of target names
714 :rtype: list
715 """
716 from tuf.repository_tool import load_repository
717 repository = load_repository(self._master_repo_dir)
718 # pylint: disable=no-member
719 return [p.lstrip(path.sep) for p in repository.targets.target_files]
721 @_master_repo_locked
722 def get_expirations(self):
723 """
724 Return the expiration dates of the TUF metadata.
726 :returns: A dictionary containing `datetime <https://docs.python.org/2/library/datetime.html#datetime.datetime>`_ values for the keys ``root``, ``targets``, ``snapshot`` and ``timestamp``.
727 :rtype: dict
728 """
729 from tuf.repository_tool import load_repository
730 repository = load_repository(self._master_repo_dir)
731 # pylint: disable=no-member
732 return {
733 'root': repository.root.expiration,
734 'targets': repository.targets.expiration,
735 'snapshot': repository.snapshot.expiration,
736 'timestamp': repository.timestamp.expiration
737 }
739class DTufCopy(_DTufCommon):
740 """
741 Class for downloading data from repositories in a Docker registry using
742 `The Update Framework <https://github.com/theupdateframework/tuf>`_ (TUF).
743 """
744 # pylint: disable=too-many-arguments
745 def __init__(self, host, repo, repos_root=None,
746 auth=None, insecure=False, auth_host=None):
747 """
748 :param host: Host name of registry. Can contain port numbers. e.g. ``registry-1.docker.io``, ``localhost:5000``.
749 :type host: str
751 :param repo: Name of the repository to access on the registry. Typically this is of the form ``username/reponame`` but for your own registries you don't actually have to stick to that. The repository is used to retrieve data and TUF metadata describing the data.
752 :type repo: str
754 :param repos_root: Directory under which to store TUF metadata. Note that the value of ``repo`` and the literal string ``copy`` are appended to this directory name before storing the metadata. Defaults to ``dtuf_repos`` in the current working directory.
755 :type repos_root: str
757 :param auth: Authentication function to be called whenever authentication to the registry is required. Receives the :class:`DTufBase` object and a HTTP response object. It should call :meth:`DTufBase.authenticate` with a username, password and ``response`` before it returns.
758 :type auth: function(dtuf_obj, response)
760 :param insecure: Use HTTP instead of HTTPS (which is the default) when connecting to the registry.
761 :type insecure: bool
763 :param auth_host: Host to use for token authentication. If set, overrides host returned by then registry.
764 :type auth_host: str
765 """
766 super(DTufCopy, self).__init__(host, repo, repos_root,
767 auth, insecure, auth_host)
768 self._copy_dir = path.join(self._repo_root, 'copy')
769 self._copy_repo_lock = fasteners.process_lock.InterProcessLock(
770 path.join(self._copy_dir, 'lock'))
771 self._copy_repo_dir = path.join(self._copy_dir, 'repository')
772 self._copy_targets_dir = path.join(self._copy_repo_dir, 'targets')
773 self._repository_mirrors = {
774 'dtuf': {
775 'url_prefix': 'https://' + host + '/' + repo,
776 'metadata_path': '',
777 'targets_path': '',
778 'confined_target_dirs': ['']
779 }
780 }
782 # pylint: disable=too-many-locals
783 @_copy_repo_locked
784 def pull_metadata(self, root_public_key=None, progress=None):
785 """
786 Download TUF metadata from the repository.
788 The metadata is checked for expiry and verified against the root public
789 key for the repository.
791 You only need to supply the root public key once, and you should obtain
792 it from the person who uploaded the metadata.
794 Target data is not downloaded - use :meth:`pull_target` for that.
796 :param root_public_key: PEM-encoded root public key. Obtain this from the repository's owner, who generates the key using :meth:`DTufMaster.create_root_key` on the repository master.
797 :type root_public_key: str
799 :param progress: Optional function to call as the download progresses. The function will be called with the hash of the metadata currently being download, the blob just read from the repository and the total size of the metadata.
800 :type progress: function(dgst, chunk, total)
802 :returns: List of targets which have been updated since you last downloaded them (using :meth:`pull_target`).
803 :rtype: list
804 """
805 import tuf.keydb
806 import tuf.roledb
807 import tuf.client.updater
808 import tuf.formats
809 import tuf.sig
810 from tuf.exceptions import BadSignatureError
811 # pylint: disable=global-statement
812 global _updater_progress
813 _updater_progress = progress
814 for d in ['current', 'previous']:
815 try:
816 makedirs(path.join(self._copy_repo_dir, 'metadata', d))
817 except OSError as exception:
818 import errno
819 if exception.errno != errno.EEXIST: 819 ↛ 820line 819 didn't jump to line 820, because the condition on line 819 was never true
820 raise
821 # If root public key was passed, we shouldn't rely on the current
822 # version but instead retrieve a new one and verify it using
823 # the public key.
824 if root_public_key:
825 dgst = self._dxf.get_alias('root.json')[0]
826 temp_file = tempfile.TemporaryFile() # pylint: disable=consider-using-with
827 try:
828 it, size = self._dxf.pull_blob(dgst, size=True)
829 _write_with_progress(it, dgst, size, temp_file, progress)
830 temp_file.seek(0)
831 metadata = temp_file.read()
832 metadata_signable = json.loads(metadata.decode('utf-8'))
833 tuf.formats.check_signable_object_format(metadata_signable)
834 # pylint: disable=protected-access
835 f = tuf.client.updater.Updater._ensure_not_expired
836 f = getattr(f, '__func__', f)
837 f(None, metadata_signable['signed'], 'root')
838 # This metadata is claiming to be root.json
839 # Get the keyid of the signature and use it to add the root
840 # public key to the keydb. Thus when we verify the signature
841 # the root public key will be used for verification.
842 keyid = metadata_signable['signatures'][0]['keyid']
843 tuf.keydb.add_key({
844 'keytype': 'rsa',
845 'scheme': metadata_signable['signed']['keys'][keyid]['scheme'],
846 'keyid': keyid,
847 'keyval': {'public': root_public_key}
848 }, keyid)
849 tuf.roledb.add_role('root', {
850 'keyids': [keyid],
851 'threshold': 1
852 })
853 if not tuf.sig.verify(metadata_signable, 'root'):
854 raise BadSignatureError('root')
855 securesystemslib.util.persist_temp_file(
856 temp_file,
857 path.join(self._copy_repo_dir,
858 'metadata',
859 'current',
860 'root.json'))
861 except:
862 temp_file.close()
863 raise
864 updater = tuf.client.updater.Updater('repository',
865 self._repository_mirrors)
866 if hasattr(updater, 'fetcher'): 866 ↛ 867line 866 didn't jump to line 867, because the condition on line 866 was never true
867 updater.fetcher._sessions = _tuf_sessions # pylint: disable=protected-access,no-member
868 updater.refresh(False)
869 targets = updater._targets_of_role('targets', skip_refresh=True) # pylint: disable=protected-access
870 updated_targets = updater.updated_targets(
871 targets, self._copy_targets_dir)
872 if path.isdir(self._copy_targets_dir):
873 targets = {t['filepath']: True for t in targets}
874 for t in listdir(self._copy_targets_dir):
875 if t not in targets:
876 remove(path.join(self._copy_targets_dir, t))
877 return [t['filepath'] for t in updated_targets]
879 @_copy_repo_locked
880 def _get_digests(self, target, sizes=False):
881 import tuf.client.updater
882 updater = tuf.client.updater.Updater('repository',
883 self._repository_mirrors)
884 if hasattr(updater, 'fetcher'): 884 ↛ 885line 884 didn't jump to line 885, because the condition on line 884 was never true
885 updater.fetcher._sessions = _tuf_sessions # pylint: disable=protected-access,no-member
886 tgt = updater.get_one_valid_targetinfo(target)
887 updater.download_target(tgt, self._copy_targets_dir)
888 with open(path.join(self._copy_targets_dir, target), 'rb') as f:
889 manifest = f.read().decode('utf-8')
890 return self._dxf.get_alias(manifest=manifest, sizes=sizes)
892 def pull_target(self, target, digests_and_sizes=False):
893 """
894 Download a target (data) from the repository.
896 Target data consists of one or more separate blobs (depending on how
897 many were uploaded). Because this function returns an iterator, download
898 of each blob occurs lazily.
900 Target information is stored in the TUF metadata, so you should have
901 called :meth:`pull_metadata` previously.
903 :param target: Name of the target to download.
904 :type target: str
906 :param digests_and_sizes: Whether to return the hash and size of each downloaded blob as well.
907 :type digests_and_sizes: bool
909 :returns: If ``digests_and_sizes`` is falsey, an iterator which yields for each blob an iterator over its content. If ``digests_and_sizes`` is truthy, an iterator which yields for each blob a tuple containing an iterator over its content, the hash of its content and its size.
910 :rtype: iterator
911 """
912 for dgst in self._get_digests(target):
913 if digests_and_sizes:
914 it, size = self._dxf.pull_blob(dgst, size=True)
915 yield it, dgst, size
916 else:
917 yield self._dxf.pull_blob(dgst)
919 def blob_sizes(self, target):
920 """
921 Return the sizes of all the blobs which make up a target.
923 :param target: Name of target
924 :type target: str
926 :returns: List of blob sizes
927 :rtype: list
928 """
929 return [size for _, size in self._get_digests(target, sizes=True)]
931 def check_target(self, target, *filenames):
932 """
933 Check whether the hashes of a target's blobs match the hashes of a
934 given list of filenames.
936 Raises `dxf.exceptions.DXFDigestMismatchError` if they don't.
938 :param target: Name of target to check.
939 :type target: str
941 :param filenames: Names of files to check against.
942 :type filenames: list
943 """
944 blob_dgsts = self._get_digests(target)
945 if len(blob_dgsts) != len(filenames):
946 raise dxf.exceptions.DXFDigestMismatchError(filenames, blob_dgsts)
947 for i, filename in enumerate(filenames):
948 file_dgst = hash_file(filename)
949 if file_dgst != blob_dgsts[i]:
950 raise dxf.exceptions.DXFDigestMismatchError(file_dgst, blob_dgsts[i])
952 @_copy_repo_locked
953 def list_targets(self):
954 """
955 Return the names of all the targets defined in the local copy of the
956 TUF metadata.
958 :returns: List of target names
959 :rtype: list
960 """
961 import tuf.client.updater
962 updater = tuf.client.updater.Updater('repository',
963 self._repository_mirrors)
964 if hasattr(updater, 'fetcher'): 964 ↛ 965line 964 didn't jump to line 965, because the condition on line 964 was never true
965 updater.fetcher._sessions = _tuf_sessions # pylint: disable=protected-access,no-member
966 return [t['filepath'] for t in updater._targets_of_role('targets', skip_refresh=True)] # pylint: disable=protected-access
968 @_copy_repo_locked
969 def get_expirations(self):
970 """
971 Return the expiration dates of the local TUF metadata copy.
973 :returns: A dictionary containing `datetime <https://docs.python.org/2/library/datetime.html#datetime.datetime>`_ values for the keys ``root``, ``targets``, ``snapshot`` and ``timestamp``.
974 :rtype: dict
975 """
976 import tuf.client.updater
977 updater = tuf.client.updater.Updater('repository',
978 self._repository_mirrors)
979 if hasattr(updater, 'fetcher'): 979 ↛ 980line 979 didn't jump to line 980, because the condition on line 979 was never true
980 updater.fetcher._sessions = _tuf_sessions # pylint: disable=protected-access,no-member
981 metadata = updater.metadata['current']
982 return {
983 'root': iso8601.parse_date(metadata['root']['expires']),
984 'targets': iso8601.parse_date(metadata['targets']['expires']),
985 'snapshot': iso8601.parse_date(metadata['snapshot']['expires']),
986 'timestamp': iso8601.parse_date(metadata['timestamp']['expires'])
987 }