PNG
IHDR x sBIT|d pHYs + tEXtSoftware www.inkscape.org< ,tEXtComment
File Manager
File Manager
Viewing File: parallel_tracker_file.py
# -*- coding: utf-8 -*-
# Copyright 2015 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper functions for composite upload tracker file functionality."""
from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals
from collections import namedtuple
import errno
import json
import random
import six
import gslib
from gslib.exception import CommandException
from gslib.tracker_file import (WriteJsonDataToTrackerFile,
RaiseUnwritableTrackerFileException)
from gslib.utils.constants import UTF8
ObjectFromTracker = namedtuple('ObjectFromTracker', 'object_name generation')
class _CompositeUploadTrackerEntry(object):
"""Enum class for composite upload tracker file JSON keys."""
COMPONENTS_LIST = 'components'
COMPONENT_NAME = 'component_name'
COMPONENT_GENERATION = 'component_generation'
ENC_SHA256 = 'encryption_key_sha256'
PREFIX = 'prefix'
def ReadParallelUploadTrackerFile(tracker_file_name, logger):
"""Read the tracker file from the last parallel composite upload attempt.
If it exists, the tracker file is of the format described in
WriteParallelUploadTrackerFile or a legacy format. If the file doesn't exist
or is formatted incorrectly, then the upload will start from the beginning.
This function is not thread-safe and must be protected by a lock if
called within Command.Apply.
Args:
tracker_file_name: The name of the tracker file to read parse.
logger: logging.Logger for outputting log messages.
Returns:
enc_key_sha256: Encryption key SHA256 used to encrypt the existing
components, or None if an encryption key was not used.
component_prefix: String prefix used in naming the existing components, or
None if no prefix was found.
existing_components: A list of ObjectFromTracker objects representing
the set of files that have already been uploaded.
"""
enc_key_sha256 = None
prefix = None
existing_components = []
tracker_file = None
# If we already have a matching tracker file, get the serialization data
# so that we can resume the upload.
try:
tracker_file = open(tracker_file_name, 'r')
tracker_data = tracker_file.read()
tracker_json = json.loads(tracker_data)
enc_key_sha256 = tracker_json[_CompositeUploadTrackerEntry.ENC_SHA256]
prefix = tracker_json[_CompositeUploadTrackerEntry.PREFIX]
for component in tracker_json[_CompositeUploadTrackerEntry.COMPONENTS_LIST]:
existing_components.append(
ObjectFromTracker(
component[_CompositeUploadTrackerEntry.COMPONENT_NAME],
component[_CompositeUploadTrackerEntry.COMPONENT_GENERATION]))
except IOError as e:
# Ignore non-existent file (happens first time a upload is attempted on an
# object, or when re-starting an upload after a
# ResumableUploadStartOverException), but warn user for other errors.
if e.errno != errno.ENOENT:
logger.warn(
'Couldn\'t read upload tracker file (%s): %s. Restarting '
'parallel composite upload from scratch.', tracker_file_name,
e.strerror)
except (KeyError, ValueError) as e:
# Legacy format did not support user-supplied encryption.
enc_key_sha256 = None
(prefix, existing_components) = _ParseLegacyTrackerData(tracker_data)
finally:
if tracker_file:
tracker_file.close()
return (enc_key_sha256, prefix, existing_components)
def _ParseLegacyTrackerData(tracker_data):
"""Parses a legacy parallel composite upload tracker file.
Args:
tracker_data: Legacy tracker file contents.
Returns:
component_prefix: The prefix used in naming the existing components, or
None if no prefix was found.
existing_components: A list of ObjectFromTracker objects representing
the set of files that have already been uploaded.
"""
# Old tracker files used a non-JSON format.
# The first line represents the prefix, followed by line pairs of object_name
# and generation. Discard the last blank line.
old_tracker_data = tracker_data.split('\n')[:-1]
prefix = None
existing_components = []
if old_tracker_data:
prefix = old_tracker_data[0]
i = 1
while i < len(old_tracker_data) - 1:
(name, generation) = (old_tracker_data[i], old_tracker_data[i + 1])
if not generation:
# Cover the '' case.
generation = None
existing_components.append(ObjectFromTracker(name, generation))
i += 2
return (prefix, existing_components)
def ValidateParallelCompositeTrackerData(tracker_file_name, existing_enc_sha256,
existing_prefix, existing_components,
current_enc_key_sha256, bucket_url,
command_obj, logger, delete_func,
delete_exc_handler):
"""Validates that tracker data matches the current encryption key.
If the data does not match, makes a best-effort attempt to delete existing
temporary component objects encrypted with the old key.
Args:
tracker_file_name: String file name of tracker file.
existing_enc_sha256: Encryption key SHA256 used to encrypt the existing
components, or None if an encryption key was not used.
existing_prefix: String prefix used in naming the existing components, or
None if no prefix was found.
existing_components: A list of ObjectFromTracker objects representing
the set of files that have already been uploaded.
current_enc_key_sha256: Current Encryption key SHA256 that should be used
to encrypt objects.
bucket_url: Bucket URL in which the components exist.
command_obj: Command class for calls to Apply.
logger: logging.Logger for outputting log messages.
delete_func: command.Apply-callable function for deleting objects.
delete_exc_handler: Exception handler for delete_func.
Returns:
prefix: existing_prefix, or None if the encryption key did not match.
existing_components: existing_components, or empty list if the encryption
key did not match.
"""
if six.PY3:
if isinstance(existing_enc_sha256, str):
existing_enc_sha256 = existing_enc_sha256.encode(UTF8)
if isinstance(current_enc_key_sha256, str):
current_enc_key_sha256 = current_enc_key_sha256.encode(UTF8)
if existing_prefix and existing_enc_sha256 != current_enc_key_sha256:
try:
logger.warn(
'Upload tracker file (%s) does not match current encryption '
'key. Deleting old components and restarting upload from '
'scratch with a new tracker file that uses the current '
'encryption key.', tracker_file_name)
components_to_delete = []
for component in existing_components:
url = bucket_url.Clone()
url.object_name = component.object_name
url.generation = component.generation
command_obj.Apply(
delete_func,
components_to_delete,
delete_exc_handler,
arg_checker=gslib.command.DummyArgChecker,
parallel_operations_override=command_obj.ParallelOverrideReason.SPEED)
except: # pylint: disable=bare-except
# Regardless of why we can't clean up old components, need to proceed
# with the user's original intent to upload the file, so merely warn.
component_names = [
component.object_name for component in existing_components
]
logger.warn(
'Failed to delete some of the following temporary objects:\n%s\n'
'(Continuing on to re-upload components from scratch.)',
'\n'.join(component_names))
# Encryption keys have changed, so the old components and prefix
# cannot be used.
return (None, [])
return (existing_prefix, existing_components)
def GenerateComponentObjectPrefix(encryption_key_sha256=None):
"""Generates a random prefix for component objects.
Args:
encryption_key_sha256: Encryption key SHA256 that will be used to encrypt
the components. This is hashed into the prefix to avoid collision
during resumption with a different encryption key.
Returns:
String prefix for use in the composite upload.
"""
return str(
(random.randint(1, (10**10) - 1) + hash(encryption_key_sha256)) % 10**10)
def WriteComponentToParallelUploadTrackerFile(tracker_file_name,
tracker_file_lock,
component,
logger,
encryption_key_sha256=None):
"""Rewrites an existing tracker file with info about the uploaded component.
Follows the format described in _CreateParallelUploadTrackerFile.
Args:
tracker_file_name: Tracker file to append to.
tracker_file_lock: Thread and process-safe Lock protecting the tracker file.
component: ObjectFromTracker describing the object that was uploaded.
logger: logging.Logger for outputting log messages.
encryption_key_sha256: Encryption key SHA256 for use in this upload, if any.
"""
with tracker_file_lock:
(existing_enc_key_sha256, prefix,
existing_components) = (ReadParallelUploadTrackerFile(
tracker_file_name, logger))
if existing_enc_key_sha256 != encryption_key_sha256:
raise CommandException(
'gsutil client error: encryption key SHA256 (%s) in tracker file '
'does not match encryption key SHA256 (%s) of component %s' %
(existing_enc_key_sha256, encryption_key_sha256,
component.object_name))
newly_completed_components = [component]
completed_components = existing_components + newly_completed_components
WriteParallelUploadTrackerFile(tracker_file_name,
prefix,
completed_components,
encryption_key_sha256=encryption_key_sha256)
def WriteParallelUploadTrackerFile(tracker_file_name,
prefix,
components,
encryption_key_sha256=None):
"""Writes information about components that were successfully uploaded.
The tracker file is serialized JSON of the form:
{
"encryption_key_sha256": sha256 hash of encryption key (or null),
"prefix": Prefix used for the component objects,
"components": [
{
"component_name": Component object name,
"component_generation": Component object generation (or null),
}, ...
]
}
where N is the number of components that have been successfully uploaded.
This function is not thread-safe and must be protected by a lock if
called within Command.Apply.
Args:
tracker_file_name: The name of the parallel upload tracker file.
prefix: The generated prefix that used for uploading any existing
components.
components: A list of ObjectFromTracker objects that were uploaded.
encryption_key_sha256: Encryption key SHA256 for use in this upload, if any.
"""
if six.PY3:
if isinstance(encryption_key_sha256, bytes):
encryption_key_sha256 = encryption_key_sha256.decode('ascii')
tracker_components = []
for component in components:
tracker_components.append({
_CompositeUploadTrackerEntry.COMPONENT_NAME: component.object_name,
_CompositeUploadTrackerEntry.COMPONENT_GENERATION: component.generation
})
tracker_file_data = {
_CompositeUploadTrackerEntry.COMPONENTS_LIST: tracker_components,
_CompositeUploadTrackerEntry.ENC_SHA256: encryption_key_sha256,
_CompositeUploadTrackerEntry.PREFIX: prefix
}
WriteJsonDataToTrackerFile(tracker_file_name, tracker_file_data)
b IDATxytVսϓ22 A@IR :hCiZ[v*E:WũZA ^dQeQ @ !jZ'>gsV仿$|?g)&x-E IENT ; @x T.i%-X} SvS5.r/ UHz^_$-W" w)Ɗ/ @Z &IoX P$K} JzX:;` &, ŋui,e6mX
Եr Kb1ԗ) D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A D A ݀!I*]R;I2$eZ#ORZSrr 6mteffu*((Pu'v{ DIߔ4^pIm '77WEEE;vƎ 4- $]'RI{\I&G :IHJ
DWBB=\WRm
o$K(V 9 ABB.}jѢv `^?IOȅ} ڶmG }T#FJ`5 6$- ھ}FI&v;0(h; Б 38CӧOWf!; A
i:F _m9s&|q%=# wZprrrla
A & P\\СC[A#! { olF}
`E2} MK/vV )i{ 4BffV\|ۭX `b@kɶ@%i$K z5zhmX [IXZ ` 'b%$r5 M4º /l
ԃ ߖ xhʔ)[@=}
K6IM}^ 5k㏷݆z
ΗÿO:gdGBmyT/@+Vɶ纽zl.yit뭷zV 0[Y^>Wsqs}\/@$(T7f.Inݺi R$푔n.~?H))\Z RW'Mo~v
Ov6oԃ xz! S,&xm/yɞԟ?'uaSѽb, 8GלKboi&3 t7Y,)JJ c[nzӳdE&KsZLӄ I ?@ & %ӟ۶mSMMњ0 iؐSZ, |J+N
~,0A 0!5%Q-YQQa 3 }$_vVr f9f?S 8` z D A D A D A D A D A D A D A D A D A dqP,تmMmg 1V?rSI꒟]u|l
RCyEf٢9 jURbztѰ!m5~tGj2DhG*{H9)꒟ר3:(+3\?/;TUݭʴ~S6lڧUJ*i$d(#=Yݺd{,p|3B)) q:vN0Y.jkק6;SɶVzHJJЀ-utѹսk>QUU \~]fFnK?&ߡ5b=z9)^|u_k-[ y%ZNU6 7Mi:]ۦ tk[n
X(e6 Bb."8cۭ|~teuuw|ήI-5"~Uk;ZicEmN/:]M>
cQ^uiƞ??Ң pc# TUU3UakNwA`:Y_V-8.KKfRi tv*
9S6ֿj,ՃNOMߤ]z^fOh|<>@Å5 _/Iu?{SY4hK/2]4%it5q]GGe2%iR|
W&f*^]??vq[LgE_3f}Fxu~}qd-ږFxu~I N >\;͗O֊:̗WJ@ Bh W=y|GgwܷH_NY?)Tdi'?խwhlmQi !SUUsw4kӺe4rfxu-[nHtMFj}H_u~w>)oV}(T'ebʒv3_[+vn @Ȭ\S}ot}w=k HFnxg S 0eޢm~l}uqZfF oZuuEg
`zt~? b;t%>WTkķh[ 2eG8LIWx,^\thrl^Ϊ{=dž<}qV@ ⠨Wy^LF_>0UkD uʫuCs$)I v:IK; 6ֲ4{^6 եm+l 3>݆uM 9u ?>Zc }g~qhKwڭ e FMM~pМuqǿz6Tb@8 @Y|jx](^]gf}M"tG
-w.@ vOqh~/HII` S[l.6nØXL9vU cOoB\xoǤ'T&I ǍQw_wpv[kmO{w~>#=P1Pɞa-we:iǏlHo꒟f9SzH?+shk%Fs:q VhqY `jvO'ρ?PyX3lх]˾uV{ݞ]1,MzYNW~̈́ joYn}ȚF߾mS]F z+EDxm/d{F{-W-4wY듏:??_gPf
^3ecg ҵs8R2מz@TA NGj)}CNi/R~}c:5{! ZHӋӾ6}T] G]7W6^n
9*,YqOZj :P?Q D FL|?-^.Ɵ7}fFhxe2Ps cz1&5\cn[=Vn[ĶE鎀uˌd3GII k;lNmشOuuRVfBE]ۣe Ӷu
:X-[(er4~LHi6:Ѻ@ԅr ST0 trk%$Č0 ez" *z " T/X9|8.C5Feg} C Q%͞ˣJvL/? j^ h & 9xF`њZ( & yF&Iݻfg #W;3^{Wo^4'v V[[K';+mӍִ] AC @ W?1^{එyh +^]fm~iԵ] AB @WTk̏t uR?l.OIHiYyԶ] Aˀ 7c:q}ힽaf6Z~қm(+sK4{^6}T *UUu]n.:kx{:2 _m=sAߤU@? Z-Vކе z왍Nэ{|5 pڶn b
p-@sPg]0G7fy-M{GCF'%{4` =$-Ge\ eU:m+Zt'WjO!OAF @ik&t݆ϥ_
e}=]" Wz_.͜E3leW Fih|t-wZۍ-uw=6 YN {6|} | *={Ѽn.S .z1z jۻTH]흾 D uDvmvK .` V]yY~sI@t?/ϓ. m&[ " +P?MzovVЫG3- G RR[(!!\_,^%?v@ ҵő
m`Y)tem8GM x.))A]Y i`ViW`?^~! S#^+ѽGZj?Vģ 0.))AlzL*]OXrY `DBBLOj{-MH'ii-ϰ ok7^
)쭡b]UXSְmռY|5 *cֽk0B7镹%ڽP# 8nȎq}mJr23_>lE5$iwui+ H~F`IjƵ@q
\ @#qG 0" . 0" l ` . 0! , AQHN6qzkKJ#o;`Xv2>,tێJJ7Z/*A .@fفjMzkg @TvZH3Zxu6Ra'%O?/d Q5x YkU]Rֽkق@ DaS^RSּ5|BeHNN͘p
HvcYcC5:y
#`οb; z2 .!kr}gUWkyZn=f Pvsn3p~;4p˚=ē~NmI] ¾ 0lH[_L hs h_ ғߤc_њe c) g 7VIZ5 yrgk̞W#IjӪv>՞y睝M8[ |]\շ 8M 6%|@P ZڨI-m>=k= 'aiRo-x?>Q. }`Ȏ:Wsmu u >
.@,&;+!!˱tﭧD QwRW\vF\~Q7>sp Yw$%A~;~}6 ¾ g &if_=j,v+U L1(tW ake:@Ș>j$Gq2t7S?vL|]u/ .(0 E 6Mk6hiۺzښOrifޱxm/Gx> Lal%%~{lBsR4* }{0Z/ tNIɚpV ^# Lf:u@k#RSu
=S^ZyuR/.@n&z~B=0eg뺆#,Þ[ B/?H uUf7y
Wy}Bwegל`Wh(||`l`.;Ws?V@" c:iɍL֯PGv6zctM̠':wuW;d=;EveD}9J@ B(0iհ bvP1{\P&G7DIy_$-Q jm~Yrr&]CDv%bh|Yzni _ R ;kg}nJOIIwy uL}{ЌNj}:+3Y?:WJ/N+Rzd=hb;dj͒suݔ@NKMԄ jqzC5 @y°hL m;*5ezᕏ=ep
XL n?מ:r`۵tŤZ|1v`V뽧_csج'ߤ%oTuumk%%%h)uy]Nk[n
'b2 l.=͜E%gf$[c;s:V-͞WߤWh-j7]4=F-X]>ZLSi[Y*We; Zan(Ӈ W|e(HNNP5[=
r4tP
&0< pc# `vTNV
GFqvTi * Tyam$ߏWyE* VJKMTfFw >'$-ؽ.Ho.8c"@ D A D A D A D A D A D A D A D A D A~j*֘, N;Pi3599h=g oض L giJ5փy~}&Zd9 p֚
e:|hL``b/d9 p? fgg+%%hMg Xosج, ΩOl0Zh=x djL mhݻo O [g_l, 8a]٭+ӧ0 $I] c]:粹:Teꢢ"5a^Kgh,&= =՟ ^߶ߢE ܹS J}I%:8
IDAT~,9/ʃ PW'Mo }zNƍ쨓zPb NZ~^z=4mswg;5 Y~SVMR XUյڱRf?s:w
; 6H:º i 5 -maM &O3; 1IKeam Zh͛7+##v+c ~u~ca] GnF'ټL~ PPPbn
v oC 4R,ӟgg %hq} @#M4IÇ Oy^xM Zx
) yO w @HkN˖-Sǎ mb]X@n+i͖ !++K3gd\$mt$^ YfJ \8PRF )77Wא!Cl $i: @@_oG I{$# 8磌ŋ9 1A (Im7 ֭>}ߴJq7ޗt^ -[ԩSj*}%]&'
- ɓ'ꫯVzzvB#; a
7@GxI{jƌ.LÇWBB7 `O"I$ /@R @ eee@۷ >}0 ,ɒ2$53Xs|cS~r pTYYY} k Hc %&k.], @ A D A D A D A D A D A D A D A D A @lT<%''*Lo^={رc5h %$+CnܸQ3fҥK}vUVVs9G
R,_{ xˇ3 o߾;TTTd}馛]uuuG~iԩ@4 bnvmvfϞ /Peeeq}}z a
I~,誫{UWW뮻}_~YƍSMMMYχ ֝waw \ ď cxꩧtEƍկ_?۷5@u ?1kNׯWzz/wy>}zj3 k (ٺu q_Zvf̘:~ AB Q&r| ! %KҥKg Ԟ={<_X-z ! CyFUUz~ AB QIIIjݺW $ UXXDٳZ~ AB QƍecW $<(~<RSSvZujjjԧO ZQu@4 8m&&&jԩg $ ď 1h ͟?_{768@ g
=@ `)))5o6m 3 ) ѣƌJ;wҿUTT /KZR{~a=@ 0o<*狔iFɶ[ˎ ;T]] OX@ ?K.ۈxN p p p p p p p p p p p p p p p p pP fl߾] ,{ｧk۶mڿo5 BTӦMӴiӴ|r D B2e |An!D y'tkΝ[A $***t5'
" !駟oaD nΝ:t֭[gD ШQ0 6qD;@ x M6v( PiizmZ 4ew" @ ̴ixf [~-F ٱc&IZ 2|n!?$ @{[HTɏ #@h Ȏ I# _m (F /6Z3 z'\r, r!;w2Z3j=~G Y 7"I$i I. p_" ? p N` y D D ?: _ G ÿa b7 J ! Bx @0 Bo
cG @`1C [ @0G
@`0C _ u V1 aC X > W ` | `! < S `" <. `# c `? cA C 4
? c p# ~ @0 ?: 0 8& _ M Q1 J h# ? /` 7; I q 7 a w Q A 1 H p
! #< 8/# @1Ul7 =S=K.4Z?E _$i@ ! 1!E 4? ` P_ @B ă 1 0# : " aU,xb F Y1
[n |n
#' v EH:`x b
# v D 4 Y hi.i& EΖv# O H 4IŶ}:Ik h @tZRF # (tXҙzZ ? I3l7 q@õ|ۍ 1,GpuY Ꮿ@ hJv# x xk$
v# 9 5 }_$ c S#=+"K {F*m7`# %H:NRS p6I?sIՖ{Ap$I$I:QRv2$Z @UJ*$]<FO4 IENDB`